diff --git a/lib/server/domain/account-status.ts b/lib/server/domain/account-status.ts index 0fce907..5fbb66b 100644 --- a/lib/server/domain/account-status.ts +++ b/lib/server/domain/account-status.ts @@ -5,6 +5,7 @@ import { type CredentialRecord, } from './credentials'; import { getModelsForCredential } from '../proxy/codebuddy'; +import { asRecord } from '../shared/content'; export interface AccountStatusSnapshot { checkin: { claimed: boolean | null; message: string | null }; @@ -26,11 +27,6 @@ const getBearerToken = (credential: CredentialRecord): string => credential.data.bearer_token ?? credential.data.access_token ?? '', ).trim(); -const asRecord = (value: unknown): Record | null => - value && typeof value === 'object' - ? (value as Record) - : null; - const findValue = (value: unknown, keys: string[]): unknown => { const record = asRecord(value); if (record) { diff --git a/lib/server/domain/debug.ts b/lib/server/domain/debug.ts index c05dbbf..192a3c1 100644 --- a/lib/server/domain/debug.ts +++ b/lib/server/domain/debug.ts @@ -9,6 +9,7 @@ import { trimStorageDebugLogs, writeStorageJson, } from '../storage'; +import { asRecord } from '../shared/content'; export interface DebugLogEntry { credentialFilename: string | null; @@ -716,12 +717,6 @@ const captureIndependentResponseSnapshot = async ( }; }; -const asRecord = (value: unknown): Record | null => { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : null; -}; - const toTokenCount = (value: unknown): number => { const numeric = typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index 8bbc7f2..5e26861 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -1,1683 +1,26 @@ import type { NextRequest } from 'next/server'; -import { - getDefaultModel, - isWebFetchEnabled, - isWebSearchEnabled, -} from '../domain/config'; import type { DebugTrace } from '../domain/debug'; - -import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; import { - getServerToolStreamEvent, - getServerToolExecutions, - getServerToolTurns, - type ServerToolExecution, - type ServerToolTurn, -} from './web-search-loop'; + anthropicErrorType, + createAnthropicError, + getUpstreamErrorMessage, +} from './anthropic/errors'; import { - anthropicStreamErrorChunks, - createStreamCloser, - toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { extractErrorMessage } from '../shared/http'; + buildChatRequestBody, + shouldBridgeAnthropicServerTools, +} from './anthropic/request'; +import { mapOpenAIResponseToAnthropic } from './anthropic/response'; import { - markServerTool, - normalizeToolName, - WEB_FETCH_TOOL_NAME, - WEB_FETCH_TOOL_TYPE_PREFIX, - WEB_SEARCH_TOOL_NAME, - WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; - -const MAX_STREAM_FRAME_LENGTH = 1_000_000; - -// --------------------------------------------------------------------------- -// Anthropic Messages API types -// --------------------------------------------------------------------------- - -interface AnthropicImageSource { - type?: string; - media_type?: string; - data?: string; - url?: string; -} - -interface AnthropicContentBlock { - type: string; - text?: string; - cache_control?: { type?: string }; - id?: string; - name?: string; - input?: unknown; - thinking?: string; - /** - * Accepted on inbound blocks but never sent by us — see - * `buildThinkingBlock`. Anthropic's signatures hold an encrypted copy of the - * reasoning; a client may replay one from a session it started elsewhere, and - * we skip those rather than forward ciphertext as if it were text. - */ - signature?: string; - /** Present on `redacted_thinking` blocks, which carry no readable text. */ - data?: string; - tool_use_id?: string; - content?: unknown; - source?: AnthropicImageSource; -} - -interface AnthropicMessage { - role: 'user' | 'assistant'; - content: string | AnthropicContentBlock[]; -} - -interface AnthropicTool { - name: string; - description?: string; - input_schema: Record; - type?: string; -} - -interface AnthropicThinkingConfig { - type?: string; - budget_tokens?: number; -} - -interface AnthropicMessagesRequestBody { - model?: string; - messages?: AnthropicMessage[]; - system?: string | AnthropicContentBlock[]; - max_tokens?: number; - temperature?: number; - top_p?: number; - top_k?: number; - stop_sequences?: string[]; - stream?: boolean; - tools?: AnthropicTool[]; - tool_choice?: unknown; - thinking?: AnthropicThinkingConfig; - metadata?: Record; -} - -// --------------------------------------------------------------------------- -// OpenAI response types (mirrors of codebuddy.ts internals) -// --------------------------------------------------------------------------- - -interface OpenAIToolCall { - index?: number; - id?: string; - type?: string; - function?: { - arguments?: string; - name?: string; - }; -} - -interface OpenAIChatMessage { - role?: string; - content?: unknown; - tool_calls?: OpenAIToolCall[]; - reasoning_content?: string; - reasoning?: string; -} - -interface OpenAIChatChoice { - index?: number; - message?: OpenAIChatMessage; - delta?: OpenAIChatMessage; - finish_reason?: string | null; -} - -interface OpenAIStreamError { - error?: { message?: string; status?: number }; -} - -interface OpenAIUsage { - prompt_tokens?: number; - completion_tokens?: number; - total_tokens?: number; - prompt_tokens_details?: { - cached_tokens?: number; - cache_creation_tokens?: number; - }; - completion_tokens_details?: { - reasoning_tokens?: number; - }; -} - -interface OpenAIChatResponse { - id?: string; - model?: string; - choices?: OpenAIChatChoice[]; - usage?: OpenAIUsage; -} - -interface OpenAIStreamChunk { - id?: string; - model?: string; - choices?: OpenAIChatChoice[]; - usage?: OpenAIUsage; -} - -interface ChatTextBlock { - cache_control?: { type?: string }; - text: string; - type: 'text'; -} - -/** - * An image part in the OpenAI Chat shape. Emitted in this shape rather than a - * native Anthropic one because the request is translated to Chat before it - * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the - * `responses` upstream converts it to `input_image`. - */ -interface ChatImageBlock { - cache_control?: { type?: string }; - image_url: { url: string }; - type: 'image_url'; -} - -type ChatContentPart = string | ChatTextBlock | ChatImageBlock; - -type ChatContent = string | Array; - -/** - * Text-only content, used where images are not representable — the system - * prompt and the intermediate text-part buffer. - */ -type ChatTextContent = string | ChatTextBlock[]; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -const createAnthropicId = (prefix: string): string => { - return `${prefix}_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const stringifyContent = (value: unknown): string => { - if (typeof value === 'string') { - return value; - } - - if (Array.isArray(value)) { - return value - .map((item) => { - if (typeof item === 'string') { - return item; - } - - if (item && typeof item === 'object' && 'text' in item) { - return String((item as { text?: unknown }).text ?? ''); - } - - return JSON.stringify(item); - }) - .join(''); - } - - if (value === undefined || value === null) { - return ''; - } - - return JSON.stringify(value); -}; - -const mapTextPartsToChatContent = ( - parts: Array, -): ChatTextContent => { - const textParts = parts.filter((part) => - typeof part === 'string' ? part.length > 0 : part.text.length > 0, - ); - const hasStructuredText = textParts.some((part) => typeof part !== 'string'); - - if (!hasStructuredText) { - return textParts.join('\n'); - } - - return textParts.flatMap((part, index) => [ - ...(index > 0 ? [{ type: 'text' as const, text: '\n' }] : []), - typeof part === 'string' ? { type: 'text' as const, text: part } : part, - ]); -}; - -/** - * Builds the `image_url` value for an Anthropic image block. Base64 sources - * become a data URI because the upstream Chat/Responses APIs expect a URL; - * `url` sources pass through untouched. Returns undefined for an unusable - * source so the caller can fall back to a text placeholder rather than - * emitting a block the upstream would reject. - */ -const buildChatImageUrl = ( - source: AnthropicImageSource | undefined, -): string | undefined => { - if (!source || typeof source !== 'object') { - return undefined; - } - - if (source.type === 'url' || (!source.data && source.url)) { - return typeof source.url === 'string' && source.url - ? source.url - : undefined; - } - - if (typeof source.data !== 'string' || !source.data) { - return undefined; - } - - const mediaType = - typeof source.media_type === 'string' && source.media_type - ? source.media_type - : 'image/png'; - - return `data:${mediaType};base64,${source.data}`; -}; - -/** - * Like `mapTextPartsToChatContent`, but keeps image parts as real image - * blocks instead of collapsing them into text. Falls back to the text-only - * result when nothing resolved to an image. - */ -const mapContentPartsToChat = (parts: ChatContentPart[]): ChatContent => { - const hasImage = parts.some( - (part) => typeof part === 'object' && part.type === 'image_url', - ); - - if (!hasImage) { - return mapTextPartsToChatContent( - parts.filter( - (part): part is string | ChatTextBlock => - typeof part === 'string' || part.type === 'text', - ), - ); - } - - const blocks: Array = []; - let pendingText: Array = []; - - const flushText = (): void => { - if (!pendingText.length) { - return; - } - const textContent = mapTextPartsToChatContent(pendingText); - if (typeof textContent === 'string') { - blocks.push({ type: 'text', text: textContent }); - } else { - blocks.push(...textContent); - } - pendingText = []; - }; - - for (const part of parts) { - if (typeof part === 'object' && part.type === 'image_url') { - flushText(); - blocks.push(part); - continue; - } - pendingText.push(part); - } - - flushText(); - - return blocks; -}; - -const extractSystemText = ( - system: string | AnthropicContentBlock[] | undefined, -): ChatTextContent => { - if (!system) { - return ''; - } - - if (typeof system === 'string') { - return system; - } - - return mapTextPartsToChatContent( - system.map((block) => { - if (block.type === 'text') { - const text = block.text ?? ''; - - return block.cache_control - ? { type: 'text', text, cache_control: block.cache_control } - : text; - } - - return stringifyContent(block); - }), - ); -}; - -// --------------------------------------------------------------------------- -// Request translation: Anthropic → OpenAI -// --------------------------------------------------------------------------- - -interface ChatMessage { - role: string; - content: ChatContent | null; - tool_calls?: Array<{ - id: string; - type: string; - function: { - name: string; - arguments: string; - }; - }>; - tool_call_id?: string; - /** - * Prior-turn reasoning for this assistant message. Not part of the OpenAI - * schema; the CodeBuddy chat upstream round-trips it, and a provider that - * does not know the field ignores it. - */ - reasoning?: string; -} - -const decodeOpaqueServerToolContent = (value: unknown): unknown => { - if (typeof value !== 'string' || !value) { - return null; - } - - try { - const binary = atob(value); - const bytes = Uint8Array.from(binary, (character) => - character.charCodeAt(0), - ); - - return JSON.parse(new TextDecoder().decode(bytes)) as unknown; - } catch { - return null; - } -}; - -const formatAnthropicServerToolResult = ( - block: AnthropicContentBlock, -): string => { - if (block.type === 'web_search_tool_result' && Array.isArray(block.content)) { - return block.content - .map((value, index) => { - const item = - value && typeof value === 'object' - ? (value as Record) - : {}; - const decoded = decodeOpaqueServerToolContent(item.encrypted_content); - const source = - decoded && typeof decoded === 'object' - ? (decoded as Record) - : item; - const title = String(source.title ?? item.title ?? '').trim(); - const url = String(source.url ?? item.url ?? '').trim(); - const text = String( - source.content ?? source.snippet ?? source.text ?? '', - ).trim(); - - return [ - `${index + 1}. ${title || url || 'Search result'}`, - ...(url ? [`URL: ${url}`] : []), - ...(text ? [text] : []), - ].join('\n'); - }) - .join('\n\n'); - } - - if ( - block.type === 'web_fetch_tool_result' && - block.content && - typeof block.content === 'object' - ) { - const result = block.content as Record; - const document = - result.content && typeof result.content === 'object' - ? (result.content as Record) - : null; - const source = - document?.source && typeof document.source === 'object' - ? (document.source as Record) - : null; - const url = typeof result.url === 'string' ? result.url : ''; - const text = typeof source?.data === 'string' ? source.data : ''; - - return [url, text].filter(Boolean).join('\n\n'); - } - - // Nested images are emitted as real image parts by - // `collectAnthropicNestedImages`, so they are excluded here to keep their - // base64 payload out of the text. - if (Array.isArray(block.content)) { - return stringifyContent( - block.content.filter((value) => { - return !( - value && - typeof value === 'object' && - (value as AnthropicContentBlock).type === 'image' - ); - }), - ); - } - - return typeof block.content === 'string' - ? block.content - : stringifyContent(block.content); -}; - -/** - * Images nested inside a `tool_result` content array, e.g. a screenshot a tool - * returned. The outer block is handled by the `tool_result` branch, whose - * formatter stringifies nested content — so without extracting them here the - * model would receive the base64 payload as text. - */ -const collectAnthropicNestedImages = ( - block: AnthropicContentBlock, -): ChatImageBlock[] => { - if (!Array.isArray(block.content)) { - return []; - } - - return block.content.flatMap((value): ChatImageBlock[] => { - if (!value || typeof value !== 'object') { - return []; - } - - const nested = value as AnthropicContentBlock; - - if (nested.type !== 'image') { - return []; - } - - const imageUrl = buildChatImageUrl(nested.source); - - return imageUrl - ? [{ type: 'image_url', image_url: { url: imageUrl } }] - : []; - }); -}; - -const mapAnthropicContentToChat = ( - content: string | AnthropicContentBlock[], - role: 'user' | 'assistant', -): ChatMessage[] => { - if (typeof content === 'string') { - return [{ role, content }]; - } - - const parts: ChatContentPart[] = []; - const toolCalls: Array<{ - id: string; - type: string; - function: { - name: string; - arguments: string; - }; - }> = []; - const toolResults: ChatMessage[] = []; - const messages: ChatMessage[] = []; - /** - * Reasoning recovered from thinking blocks in this assistant message. - * - * Attached to the message the blocks belong to rather than sent on its own: - * a bare reasoning entry is not a valid chat message, and the upstream needs - * the reasoning alongside the text and tool calls it produced. - */ - let pendingReasoning = ''; - const flushAssistantMessage = (): void => { - const content = mapContentPartsToChat(parts); - const hasContent = typeof content === 'string' ? content.length > 0 : true; - - if (!toolCalls.length && !hasContent && !pendingReasoning) { - return; - } - - messages.push({ - role: 'assistant', - content: hasContent ? content : null, - ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}), - // `reasoning` is the field the CodeBuddy chat upstream round-trips. It - // is not part of the OpenAI schema, but the upstream accepts it and - // ignoring an unknown field costs nothing if it ever stops doing so. - ...(pendingReasoning ? { reasoning: pendingReasoning } : {}), - }); - parts.length = 0; - toolCalls.length = 0; - pendingReasoning = ''; - }; - - for (const block of content) { - if (block.type === 'text') { - const text = block.text ?? ''; - - parts.push( - block.cache_control - ? { type: 'text', text, cache_control: block.cache_control } - : text, - ); - } else if (block.type === 'tool_use' || block.type === 'server_tool_use') { - toolCalls.push({ - id: block.id ?? createAnthropicId('toolu'), - type: 'function', - function: { - name: block.name ?? 'unknown', - arguments: JSON.stringify(block.input ?? {}), - }, - }); - } else if ( - block.type === 'tool_result' || - block.type === 'web_search_tool_result' || - block.type === 'web_fetch_tool_result' - ) { - const nestedImages = collectAnthropicNestedImages(block); - - const resultMessage: ChatMessage = { - role: 'tool', - content: nestedImages.length - ? mapContentPartsToChat([ - formatAnthropicServerToolResult(block), - ...nestedImages, - ]) - : formatAnthropicServerToolResult(block), - tool_call_id: block.tool_use_id ?? '', - }; - - if (role === 'assistant') { - flushAssistantMessage(); - messages.push(resultMessage); - } else { - toolResults.push(resultMessage); - } - } else if ( - block.type === 'thinking' || - block.type === 'redacted_thinking' - ) { - // Replaying prior-turn reasoning is required inside a tool-use turn and - // harmless elsewhere, so recover it instead of dropping it. - // - // The `thinking` field carries the reasoning. A `signature` is only ever - // read when it is one we minted on the Responses path; a genuine - // Anthropic signature is ciphertext, and forwarding it upstream would put - // gibberish where reasoning belongs. - // - // `redacted_thinking` has no readable text at all, only `data`, but must - // still be matched here: without this branch it fell through to - // `stringifyContent` and the model received a JSON dump of the opaque - // payload as if it were user prose. - const reasoning = block.thinking ?? ''; - - if (reasoning) { - pendingReasoning = pendingReasoning - ? `${pendingReasoning}${reasoning}` - : reasoning; - } - } else if (block.type === 'image' && block.source) { - // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url', - // media_type, data | url } }`. Emit a real image block so the upstream - // model sees the image; without this branch the block fell through to - // `stringifyContent` and the model received a JSON dump of the base64 - // payload as text. An `image` block with no `source` is not a real - // Anthropic image, so it keeps the generic stringified handling. - const imageUrl = buildChatImageUrl(block.source); - - parts.push( - imageUrl - ? { - type: 'image_url', - image_url: { url: imageUrl }, - // Preserve an explicit cache breakpoint, matching how text - // blocks carry `cache_control` through. Without this the - // requested breakpoint is dropped and `applyPromptCacheControl` - // falls back to its own automatic placement. - ...(block.cache_control - ? { cache_control: block.cache_control } - : {}), - } - : stringifyContent(block), - ); - } else { - parts.push(stringifyContent(block)); - } - } - - if (role === 'user') { - messages.push(...toolResults); - const content = mapContentPartsToChat(parts); - const hasContent = typeof content === 'string' ? content.length > 0 : true; - if (hasContent) { - messages.push({ role: 'user', content }); - } - } else { - flushAssistantMessage(); - } - - return messages; -}; - -const mapAnthropicMessagesToChat = ( - messages: AnthropicMessage[], -): ChatMessage[] => { - const result: ChatMessage[] = []; - - for (const msg of messages) { - const mapped = mapAnthropicContentToChat(msg.content, msg.role); - - if (mapped.length === 0) { - continue; - } - - for (const item of mapped) { - result.push({ - ...item, - }); - } - } - - return result; -}; - -const mapAnthropicToolsToChat = ( - tools: AnthropicTool[] | undefined, -): unknown[] | undefined => { - if (!tools?.length) { - return undefined; - } - - return tools.map((tool) => { - const mapped = { - type: 'function', - function: { - name: tool.name, - description: tool.description, - parameters: tool.input_schema, - }, - }; - const normalizedType = normalizeToolName(tool.type ?? ''); - const serverDeclared = [ - WEB_SEARCH_TOOL_TYPE_PREFIX, - WEB_FETCH_TOOL_TYPE_PREFIX, - ].some((prefix) => normalizedType.startsWith(normalizeToolName(prefix))); - - return serverDeclared ? markServerTool(mapped) : mapped; - }); -}; - -const shouldBridgeAnthropicServerTools = async ( - tools: AnthropicTool[] | undefined, -): Promise => { - const names = new Set( - (tools ?? []).map((tool) => normalizeToolName(tool.name)), - ); - const [searchEnabled, fetchEnabled] = await Promise.all([ - names.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) - ? isWebSearchEnabled() - : false, - names.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) - ? isWebFetchEnabled() - : false, - ]); - - return searchEnabled || fetchEnabled; -}; - -const mapAnthropicToolChoiceToChat = (toolChoice: unknown): unknown => { - if (!toolChoice || typeof toolChoice !== 'object') { - return toolChoice; - } - - const tc = toolChoice as { type?: string; name?: string }; - - if (tc.type === 'auto') { - return 'auto'; - } - - if (tc.type === 'any') { - return 'required'; - } - - if (tc.type === 'tool' && tc.name) { - return { - type: 'function', - function: { name: tc.name }, - }; - } - - if (tc.type === 'none') { - return 'none'; - } - - return toolChoice; -}; - -const buildChatRequestBody = async ( - body: AnthropicMessagesRequestBody, -): Promise> => { - const systemText = extractSystemText(body.system); - const chatMessages = mapAnthropicMessagesToChat(body.messages ?? []); - - const messages: ChatMessage[] = []; - const disableParallelToolUse = - body.tool_choice && typeof body.tool_choice === 'object' - ? (body.tool_choice as { disable_parallel_tool_use?: unknown }) - .disable_parallel_tool_use - : undefined; - - if (systemText) { - messages.push({ role: 'system', content: systemText }); - } - - messages.push(...chatMessages); - - const result: Record = { - model: - typeof body.model === 'string' && body.model.trim() - ? body.model - : await getDefaultModel('claude-sonnet-4.6'), - messages, - stream: body.stream ?? false, - max_tokens: body.max_tokens, - temperature: body.temperature, - top_p: body.top_p, - stop: body.stop_sequences, - tools: mapAnthropicToolsToChat(body.tools), - tool_choice: mapAnthropicToolChoiceToChat(body.tool_choice), - parallel_tool_calls: - typeof disableParallelToolUse === 'boolean' - ? !disableParallelToolUse - : undefined, - }; - - // Pass through thinking/reasoning config so upstream models that support - // extended thinking can honor it. - if (body.thinking) { - result.thinking = body.thinking; - } - - return result; -}; - -// --------------------------------------------------------------------------- -// Response translation: OpenAI → Anthropic (non-streaming) -// --------------------------------------------------------------------------- - -const mapOpenAIUsageToAnthropic = ( - usage: OpenAIUsage | undefined, - serverToolExecutions: ServerToolExecution[] = [], -): Record => { - const cacheCreationTokens = - usage?.prompt_tokens_details?.cache_creation_tokens ?? 0; - const cacheReadTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; - // prompt_tokens is the total prompt count including cached tokens. - // Anthropic reports cached/created tokens separately, so input_tokens - // must be the non-cache remainder to avoid double-counting. - const inputTokens = Math.max( - 0, - (usage?.prompt_tokens ?? 0) - cacheCreationTokens - cacheReadTokens, - ); - const outputTokens = usage?.completion_tokens ?? 0; - - const mapped: Record> = { - input_tokens: inputTokens, - output_tokens: outputTokens, - cache_creation_input_tokens: cacheCreationTokens, - cache_read_input_tokens: cacheReadTokens, - }; - - if (serverToolExecutions.length) { - mapped.server_tool_use = { - web_search_requests: serverToolExecutions.filter( - (execution) => execution.type === 'web_search', - ).length, - web_fetch_requests: serverToolExecutions.filter( - (execution) => execution.type === 'web_fetch', - ).length, - }; - } - - return mapped; -}; - -const encodeOpaqueServerToolContent = (value: unknown): string => { - const bytes = new TextEncoder().encode(JSON.stringify(value)); - let binary = ''; - - for (const byte of bytes) { - binary += String.fromCharCode(byte); - } - - return btoa(binary); -}; - -const buildAnthropicServerToolBlocks = ( - execution: ServerToolExecution, -): AnthropicContentBlock[] => { - const id = createAnthropicId('srvtoolu'); - const result = - execution.type === 'web_search' - ? { - type: 'web_search_tool_result', - tool_use_id: id, - content: execution.result.results.map((item) => ({ - type: 'web_search_result', - url: item.url ?? '', - title: item.title ?? '', - encrypted_content: encodeOpaqueServerToolContent(item), - })), - } - : { - type: 'web_fetch_tool_result', - tool_use_id: id, - content: { - type: 'web_fetch_result', - url: execution.result.url ?? execution.input.url, - content: { - type: 'document', - source: { - type: 'text', - media_type: 'text/plain', - data: execution.result.content, - }, - }, - }, - }; - - return [ - { - type: 'server_tool_use', - id, - name: execution.type, - input: execution.input, - }, - result, - ]; -}; - -const buildAllAnthropicServerToolBlocks = ( - executions: ServerToolExecution[], -): AnthropicContentBlock[] => - executions.flatMap(buildAnthropicServerToolBlocks); - -/** - * We do not mint a `signature` on this path. It would have to duplicate the - * `thinking` text to be replayable, which puts the reasoning on the wire twice - * for callers that count it — and the block already replays fine: Anthropic - * clients echo `thinking` back, which is what inbound handling reads. - */ -const buildThinkingBlock = (thinking: string): AnthropicContentBlock => ({ - type: 'thinking', - thinking, -}); - -/** - * Lays a server-tool turn out the way Anthropic does: each hop contributes its - * own thinking and text, followed by the tool blocks that hop triggered. - * - * `turns` carries the per-hop grouping the OpenAI-shaped payload cannot. Under - * that protocol a multi-hop turn collapses into one `content` string and one - * `reasoning_content` string, which loses where one hop's reasoning ends and the - * next begins — so the grouping has to be recovered before it is joined, which - * is why the loop emits it alongside the strings rather than this file - * reconstructing it. - * - * Anthropic's own server tools run multiple hops inside one assistant message, - * and a client replaying that message expects `[thinking] [text] [tool_use] - * [tool_result] [thinking] [text]`. Gathering the blocks by kind instead — every - * tool ahead of all the prose — puts each search before the reasoning that asked - * for it and merges hops that were never contiguous. - */ -const buildAnthropicTurnBlocks = ( - turns: ServerToolTurn[], -): AnthropicContentBlock[] => { - const blocks: AnthropicContentBlock[] = []; - - turns.forEach((turn) => { - if (turn.reasoning) { - blocks.push(buildThinkingBlock(turn.reasoning)); - } - - if (turn.text) { - blocks.push({ type: 'text', text: turn.text }); - } - - blocks.push(...buildAllAnthropicServerToolBlocks(turn.executions)); - }); - - return blocks; -}; - -const mapOpenAIResponseToAnthropic = ( - openaiResponse: OpenAIChatResponse, - model: string, - serverToolExecutions: ServerToolExecution[] = [], - turns?: ServerToolTurn[], -): Record => { - const choice = openaiResponse.choices?.[0]; - const message = choice?.message; - - // Thinking / reasoning content - const reasoningText = message?.reasoning_content ?? message?.reasoning ?? ''; - - // Text content - const textContent = - typeof message?.content === 'string' ? message.content : ''; - - // With per-hop grouping the turns already hold every block in order, prose - // included. Without it — no server tool ran, or a path that never grouped the - // hops — fall back to Anthropic's own order: thinking and text first, then - // the server-tool blocks they led to. - const contentBlocks: AnthropicContentBlock[] = turns - ? buildAnthropicTurnBlocks(turns) - : []; - - if (!turns) { - if (reasoningText) { - contentBlocks.push(buildThinkingBlock(reasoningText)); - } - - if (textContent) { - contentBlocks.push({ type: 'text', text: textContent }); - } - - contentBlocks.push( - ...buildAllAnthropicServerToolBlocks(serverToolExecutions), - ); - } - - // Tool calls - const toolCalls = message?.tool_calls ?? []; - - for (const call of toolCalls) { - let input: unknown = {}; - - try { - input = JSON.parse(call.function?.arguments ?? '{}'); - } catch { - input = {}; - } - - contentBlocks.push({ - type: 'tool_use', - id: call.id ?? createAnthropicId('toolu'), - name: call.function?.name ?? 'unknown', - input, - }); - } - - const stopReason = mapFinishReasonToAnthropic( - choice?.finish_reason, - toolCalls.length > 0, - ); - - return { - id: openaiResponse.id ?? createAnthropicId('msg'), - type: 'message', - role: 'assistant', - model, - content: contentBlocks, - stop_reason: stopReason, - stop_sequence: null, - usage: mapOpenAIUsageToAnthropic( - openaiResponse.usage, - serverToolExecutions, - ), - }; -}; - -const mapFinishReasonToAnthropic = ( - finishReason: string | null | undefined, - hasToolCalls: boolean, -): string => { - if (hasToolCalls || finishReason === 'tool_calls') { - return 'tool_use'; - } - - if (finishReason === 'length') { - return 'max_tokens'; - } - - if (finishReason === 'stop' || !finishReason) { - return 'end_turn'; - } - - return 'end_turn'; -}; - -// --------------------------------------------------------------------------- -// Response translation: OpenAI SSE → Anthropic SSE (streaming) -// --------------------------------------------------------------------------- - -interface StreamingToolUseState { - id: string; - name: string; - input: string; - index: number; - started: boolean; - blockEmitted: boolean; -} - -const mapOpenAIStreamToAnthropicSSE = ( - upstreamResponse: Response, - model: string, - options?: { - emitMessageStart?: boolean; - initialContentBlockCount?: number; - messageId?: string; - serverToolExecutions?: ServerToolExecution[]; - }, -): Response => { - if (!upstreamResponse.body) { - return new Response(null, { - status: upstreamResponse.status, - headers: { - 'Content-Type': 'text/event-stream; charset=utf-8', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }, - }); - } - - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - - const messageId = options?.messageId ?? createAnthropicId('msg'); - const serverToolExecutions = - options?.serverToolExecutions ?? getServerToolExecutions(upstreamResponse); - const serverToolUseIds = new Map(); - const toolUseStates = new Map(); - let nextToolIndex = 0; - let started = options?.emitMessageStart === false; - let thinkingStarted = false; - let thinkingBlockIndex = -1; - let textStarted = false; - let textBlockIndex = -1; - // Tracks how many content blocks (thinking + text) have been opened - // so tool_use blocks get correct sequential indices even after the - // prior blocks are closed mid-stream. - let contentBlockCount = options?.initialContentBlockCount ?? 0; - let finishReason: string | null = null; - let hasToolCalls = false; - let usage: OpenAIUsage | undefined; - - const enqueueEvent = (event: Record): void => { - controller.enqueue( - encoder.encode( - `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, - ), - ); - }; - - let controller: ReadableStreamDefaultController; - - // Close any open text/thinking block before starting a tool_use block. - // Anthropic streaming requires each block to be stopped before the next. - const closeOpenTextBlocks = (): void => { - if (thinkingStarted) { - // Anthropic emits the signature last, just before the block closes — but - // we do not send one here: the reasoning already went out as - // `thinking_delta`s, and duplicating it into a signature would put the - // text on the wire twice for callers that count it. See - // `buildThinkingBlock`. - enqueueEvent({ - type: 'content_block_stop', - index: thinkingBlockIndex, - }); - thinkingStarted = false; - } - - if (textStarted) { - enqueueEvent({ - type: 'content_block_stop', - index: textBlockIndex, - }); - textStarted = false; - } - }; - - const processChunk = (chunk: OpenAIStreamChunk): void => { - if (!started) { - started = true; - enqueueEvent({ - type: 'message_start', - message: { - id: messageId, - type: 'message', - role: 'assistant', - content: [], - model, - stop_reason: null, - stop_sequence: null, - usage: { - input_tokens: chunk.usage?.prompt_tokens ?? 0, - output_tokens: 0, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, - }, - }, - }); - } - - if (chunk.usage) { - usage = chunk.usage; - } - - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - - if (!delta) { - return; - } - - // Reasoning / thinking content - const reasoningText = delta.reasoning_content ?? delta.reasoning ?? ''; - - if (reasoningText) { - if (!thinkingStarted) { - thinkingBlockIndex = contentBlockCount; - thinkingStarted = true; - enqueueEvent({ - type: 'content_block_start', - index: thinkingBlockIndex, - content_block: { - type: 'thinking', - thinking: '', - }, - }); - contentBlockCount++; - } - - enqueueEvent({ - type: 'content_block_delta', - index: thinkingBlockIndex, - delta: { - type: 'thinking_delta', - thinking: reasoningText, - }, - }); - } - - // Text content - if (delta.content) { - if (!textStarted) { - // Close the thinking block before starting text so Anthropic - // stream consumers see properly ordered, non-overlapping blocks. - closeOpenTextBlocks(); - - textBlockIndex = contentBlockCount; - textStarted = true; - enqueueEvent({ - type: 'content_block_start', - index: textBlockIndex, - content_block: { - type: 'text', - text: '', - }, - }); - contentBlockCount++; - } - - enqueueEvent({ - type: 'content_block_delta', - index: textBlockIndex, - delta: { - type: 'text_delta', - text: delta.content, - }, - }); - } - - // Tool calls - if (delta.tool_calls?.length) { - hasToolCalls = true; - - // Anthropic streaming requires each content block to be closed - // before the next one starts. If we already opened a text or - // thinking block, close it now so the tool_use block is well-formed. - closeOpenTextBlocks(); - - for (const call of delta.tool_calls) { - const callId = call.id ?? `toolu_${nextToolIndex}`; - const key = callId; - - if (!toolUseStates.has(key)) { - const blockIndex = contentBlockCount + nextToolIndex; - - toolUseStates.set(key, { - id: callId, - name: '', - input: '', - index: blockIndex, - started: false, - blockEmitted: false, - }); - nextToolIndex++; - } - - const state = toolUseStates.get(key)!; - - // Accumulate name fragments (upstream may stream the function - // name across multiple deltas, e.g. "look" + "up"). - if (call.function?.name) { - state.name += call.function.name; - } - - // Emit content_block_start lazily — once we have a name and at - // least one arguments fragment, so the block header carries the - // full tool name instead of a partial fragment. - if (!state.blockEmitted && state.name && call.function?.arguments) { - state.blockEmitted = true; - enqueueEvent({ - type: 'content_block_start', - index: state.index, - content_block: { - type: 'tool_use', - id: state.id, - name: state.name, - input: {}, - }, - }); - } - - if (call.function?.arguments) { - state.input += call.function.arguments; - enqueueEvent({ - type: 'content_block_delta', - index: state.index, - delta: { - type: 'input_json_delta', - partial_json: call.function.arguments, - }, - }); - } - } - } - - if (choice?.finish_reason) { - finishReason = choice.finish_reason; - } - }; - - const finalize = (): void => { - // Close any remaining open text/thinking blocks. - closeOpenTextBlocks(); - - // Close tool use blocks - for (const [, state] of toolUseStates) { - // If the block start was never emitted (e.g. name-only deltas - // with no arguments), emit it now so the block is well-formed. - if (!state.blockEmitted) { - state.blockEmitted = true; - enqueueEvent({ - type: 'content_block_start', - index: state.index, - content_block: { - type: 'tool_use', - id: state.id, - name: state.name || 'unknown', - input: {}, - }, - }); - } - - enqueueEvent({ - type: 'content_block_stop', - index: state.index, - }); - } - - const stopReason = mapFinishReasonToAnthropic(finishReason, hasToolCalls); - - enqueueEvent({ - type: 'message_delta', - delta: { - stop_reason: stopReason, - stop_sequence: null, - }, - usage: mapOpenAIUsageToAnthropic(usage, serverToolExecutions), - }); - - enqueueEvent({ - type: 'message_stop', - }); - }; - - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - let streamRejected = false; - const closer = createStreamCloser(); - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - const stream = new ReadableStream({ - start: (ctrl) => { - controller = ctrl; - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - const rejectStream = ( - message = 'Upstream SSE frame exceeds the maximum size', - status?: number, - ): void => { - streamRejected = true; - enqueueEvent({ - type: 'error', - error: { - // An upstream status names the failure precisely, so it decides - // the type: 429 has to arrive as `rate_limit_error` or a client - // that retries on that type alone stops retrying an exhausted - // quota. Without one, fall back to the message: an oversized frame - // is a malformed stream (`invalid_request_error`), while an - // upstream deadline is the server failing (`api_error`, the type - // clients treat as retryable). - type: - typeof status === 'number' - ? anthropicErrorType(status) - : message.includes('did not produce output') - ? 'api_error' - : 'invalid_request_error', - message, - }, - }); - }; - - const flushFrames = (frames: string[]): void => { - for (const frame of frames) { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - rejectStream(); - return; - } - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - continue; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - try { - const chunk = JSON.parse(raw) as OpenAIStreamChunk; - const serverToolEvent = getServerToolStreamEvent(chunk); - - if (serverToolEvent?.phase === 'call') { - closeOpenTextBlocks(); - const index = contentBlockCount++; - const toolUseId = createAnthropicId('srvtoolu'); - serverToolUseIds.set(serverToolEvent.invocation.id, toolUseId); - enqueueEvent({ - type: 'content_block_start', - index, - content_block: { - type: 'server_tool_use', - id: toolUseId, - name: serverToolEvent.invocation.type, - input: {}, - }, - }); - enqueueEvent({ - type: 'content_block_delta', - index, - delta: { - type: 'input_json_delta', - partial_json: JSON.stringify( - serverToolEvent.invocation.input, - ), - }, - }); - enqueueEvent({ type: 'content_block_stop', index }); - continue; - } - - if (serverToolEvent?.phase === 'result') { - closeOpenTextBlocks(); - serverToolExecutions.push(serverToolEvent.execution); - const index = contentBlockCount++; - const resultBlock = buildAnthropicServerToolBlocks( - serverToolEvent.execution, - )[1]; - enqueueEvent({ - type: 'content_block_start', - index, - content_block: { - ...resultBlock, - tool_use_id: serverToolUseIds.get( - serverToolEvent.execution.id, - ), - }, - }); - enqueueEvent({ type: 'content_block_stop', index }); - continue; - } - - const upstreamError = chunk as OpenAIStreamError; - if (upstreamError.error?.message) { - rejectStream( - upstreamError.error.message, - upstreamError.error.status, - ); - return; - } - processChunk(chunk); - } catch { - // Skip unparseable frames - } - } - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - if (buffer.trim()) { - flushFrames([buffer]); - } - if (!streamRejected) { - finalize(); - } - releaseReader(); - controller.close(); - return; - } - - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - rejectStream(); - } else { - flushFrames(frames); - } - if (streamRejected) { - try { - await reader!.cancel(); - } finally { - releaseReader(); - controller.close(); - } - return; - } - } - }; - - void pump().catch((error) => { - if (cancelled) return; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage === null) { - closer.mark(); - controller.error(error); - return; - } - - void reader?.cancel().then( - () => undefined, - () => undefined, - ); - releaseReader(); - closer.fail(controller, anthropicStreamErrorChunks(timeoutMessage)); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - releaseReader(); - } - }, - }); - - return new Response(stream, { - status: 200, - headers: { - 'Content-Type': 'text/event-stream; charset=utf-8', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }, - }); -}; - -const createAnthropicServerToolEventStream = ( - request: NextRequest, - chatBody: Record, - model: string, - debugTrace?: DebugTrace, -): Response => { - const encoder = new TextEncoder(); - const messageId = createAnthropicId('msg'); - let activeReader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - - const stream = new ReadableStream({ - start: (controller) => { - const enqueueEvent = (event: Record): void => { - if (cancelled) return; - controller.enqueue( - encoder.encode( - `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, - ), - ); - }; - - enqueueEvent({ - type: 'message_start', - message: { - id: messageId, - type: 'message', - role: 'assistant', - content: [], - model, - stop_reason: null, - stop_sequence: null, - usage: { - input_tokens: 0, - output_tokens: 0, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, - }, - }, - }); - - const run = async (): Promise => { - const upstreamResponse = await proxyChatCompletions( - request, - chatBody as ChatRequestBody, - undefined, - debugTrace, - '/v1/messages', - // The findings reach the client as `web_search_tool_result` blocks, - // so the loop must not also fold them into the assistant text. - { emitStreamEvents: true, findingsAsStructuredBlocks: true }, - ); - - if (cancelled) { - await upstreamResponse.body?.cancel(); - return; - } - - if (!upstreamResponse.ok || !upstreamResponse.body) { - // A rate limit has to arrive as `rate_limit_error`, or a client that - // retries on that type alone will treat an exhausted quota as a - // generic failure and stop retrying — so the upstream status drives - // the event type even though the envelope is already streaming and - // the HTTP status cannot be changed. - const message = upstreamResponse.ok - ? 'Upstream request failed' - : await getUpstreamErrorMessage(upstreamResponse).catch( - () => 'Upstream request failed', - ); - - enqueueEvent({ - type: 'error', - error: { - type: anthropicErrorType(upstreamResponse.status), - message, - }, - }); - controller.close(); - return; - } - - const mappedResponse = mapOpenAIStreamToAnthropicSSE( - upstreamResponse, - model, - { - emitMessageStart: false, - initialContentBlockCount: 0, - messageId, - serverToolExecutions: [], - }, - ); - const reader = mappedResponse.body!.getReader(); - activeReader = reader; - - while (true) { - const { done, value } = await reader.read(); - if (cancelled) return; - if (done) break; - controller.enqueue(value); - } - - reader.releaseLock(); - activeReader = null; - controller.close(); - }; - - void run().catch((error) => { - if (!cancelled) controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - await activeReader?.cancel(reason); - activeReader?.releaseLock(); - activeReader = null; - }, - }); - - return new Response(stream, { - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -}; - -const getUpstreamErrorMessage = async (response: Response): Promise => { - const text = await response.text(); - if (!text) return 'Upstream CodeBuddy request failed'; - - try { - return extractErrorMessage(JSON.parse(text) as unknown) ?? text; - } catch { - return text; - } -}; + createAnthropicServerToolEventStream, + mapOpenAIStreamToAnthropicSSE, +} from './anthropic/stream'; +import type { + AnthropicMessagesRequestBody, + OpenAIChatResponse, +} from './anthropic/types'; +import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; +import { getServerToolExecutions, getServerToolTurns } from './web-search-loop'; // --------------------------------------------------------------------------- // Main handler @@ -1744,37 +87,6 @@ export const handleMessagesRequest = async ( } }; -export const anthropicErrorType = (status: number): string => - status === 401 - ? 'authentication_error' - : status === 403 - ? 'permission_error' - : status === 404 - ? 'not_found_error' - : status === 413 - ? 'request_too_large' - : status === 429 - ? 'rate_limit_error' - : status === 529 - ? 'overloaded_error' - : status >= 500 - ? 'api_error' - : 'invalid_request_error'; - -export const createAnthropicError = ( - status: number, - message: string, -): Response => { - const type = anthropicErrorType(status); - - return Response.json( - { - type: 'error', - error: { - type, - message, - }, - }, - { status }, - ); -}; +// Re-exported for the importers that reached these through this module before +// the split: `app/v1/messages/route.ts` and the test suite. +export { anthropicErrorType, createAnthropicError }; diff --git a/lib/server/proxy/anthropic/content.ts b/lib/server/proxy/anthropic/content.ts new file mode 100644 index 0000000..88df7d8 --- /dev/null +++ b/lib/server/proxy/anthropic/content.ts @@ -0,0 +1,145 @@ +import { stringifyContent } from '../../shared/content'; +import type { + AnthropicContentBlock, + AnthropicImageSource, + ChatContent, + ChatContentPart, + ChatImageBlock, + ChatTextBlock, + ChatTextContent, +} from './types'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +export const createAnthropicId = (prefix: string): string => { + return `${prefix}_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const mapTextPartsToChatContent = ( + parts: Array, +): ChatTextContent => { + const textParts = parts.filter((part) => + typeof part === 'string' ? part.length > 0 : part.text.length > 0, + ); + const hasStructuredText = textParts.some((part) => typeof part !== 'string'); + + if (!hasStructuredText) { + return textParts.join('\n'); + } + + return textParts.flatMap((part, index) => [ + ...(index > 0 ? [{ type: 'text' as const, text: '\n' }] : []), + typeof part === 'string' ? { type: 'text' as const, text: part } : part, + ]); +}; + +/** + * Builds the `image_url` value for an Anthropic image block. Base64 sources + * become a data URI because the upstream Chat/Responses APIs expect a URL; + * `url` sources pass through untouched. Returns undefined for an unusable + * source so the caller can fall back to a text placeholder rather than + * emitting a block the upstream would reject. + */ +export const buildChatImageUrl = ( + source: AnthropicImageSource | undefined, +): string | undefined => { + if (!source || typeof source !== 'object') { + return undefined; + } + + if (source.type === 'url' || (!source.data && source.url)) { + return typeof source.url === 'string' && source.url + ? source.url + : undefined; + } + + if (typeof source.data !== 'string' || !source.data) { + return undefined; + } + + const mediaType = + typeof source.media_type === 'string' && source.media_type + ? source.media_type + : 'image/png'; + + return `data:${mediaType};base64,${source.data}`; +}; + +/** + * Like `mapTextPartsToChatContent`, but keeps image parts as real image + * blocks instead of collapsing them into text. Falls back to the text-only + * result when nothing resolved to an image. + */ +export const mapContentPartsToChat = ( + parts: ChatContentPart[], +): ChatContent => { + const hasImage = parts.some( + (part) => typeof part === 'object' && part.type === 'image_url', + ); + + if (!hasImage) { + return mapTextPartsToChatContent( + parts.filter( + (part): part is string | ChatTextBlock => + typeof part === 'string' || part.type === 'text', + ), + ); + } + + const blocks: Array = []; + let pendingText: Array = []; + + const flushText = (): void => { + if (!pendingText.length) { + return; + } + const textContent = mapTextPartsToChatContent(pendingText); + if (typeof textContent === 'string') { + blocks.push({ type: 'text', text: textContent }); + } else { + blocks.push(...textContent); + } + pendingText = []; + }; + + for (const part of parts) { + if (typeof part === 'object' && part.type === 'image_url') { + flushText(); + blocks.push(part); + continue; + } + pendingText.push(part); + } + + flushText(); + + return blocks; +}; + +export const extractSystemText = ( + system: string | AnthropicContentBlock[] | undefined, +): ChatTextContent => { + if (!system) { + return ''; + } + + if (typeof system === 'string') { + return system; + } + + return mapTextPartsToChatContent( + system.map((block) => { + if (block.type === 'text') { + const text = block.text ?? ''; + + return block.cache_control + ? { type: 'text', text, cache_control: block.cache_control } + : text; + } + + return stringifyContent(block); + }), + ); +}; diff --git a/lib/server/proxy/anthropic/errors.ts b/lib/server/proxy/anthropic/errors.ts new file mode 100644 index 0000000..1cd6a13 --- /dev/null +++ b/lib/server/proxy/anthropic/errors.ts @@ -0,0 +1,49 @@ +import { extractErrorMessage } from '../../shared/http'; + +export const anthropicErrorType = (status: number): string => + status === 401 + ? 'authentication_error' + : status === 403 + ? 'permission_error' + : status === 404 + ? 'not_found_error' + : status === 413 + ? 'request_too_large' + : status === 429 + ? 'rate_limit_error' + : status === 529 + ? 'overloaded_error' + : status >= 500 + ? 'api_error' + : 'invalid_request_error'; + +export const getUpstreamErrorMessage = async ( + response: Response, +): Promise => { + const text = await response.text(); + if (!text) return 'Upstream CodeBuddy request failed'; + + try { + return extractErrorMessage(JSON.parse(text) as unknown) ?? text; + } catch { + return text; + } +}; + +export const createAnthropicError = ( + status: number, + message: string, +): Response => { + const type = anthropicErrorType(status); + + return Response.json( + { + type: 'error', + error: { + type, + message, + }, + }, + { status }, + ); +}; diff --git a/lib/server/proxy/anthropic/request.ts b/lib/server/proxy/anthropic/request.ts new file mode 100644 index 0000000..d4fb6ce --- /dev/null +++ b/lib/server/proxy/anthropic/request.ts @@ -0,0 +1,451 @@ +import { + getDefaultModel, + isWebFetchEnabled, + isWebSearchEnabled, +} from '../../domain/config'; +import { stringifyContent } from '../../shared/content'; +import { + markServerTool, + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_FETCH_TOOL_TYPE_PREFIX, + WEB_SEARCH_TOOL_NAME, + WEB_SEARCH_TOOL_TYPE_PREFIX, +} from '../../search/tool'; +import { + buildChatImageUrl, + createAnthropicId, + extractSystemText, + mapContentPartsToChat, +} from './content'; +import type { + AnthropicContentBlock, + AnthropicMessage, + AnthropicMessagesRequestBody, + AnthropicTool, + ChatContentPart, + ChatImageBlock, + ChatMessage, +} from './types'; + +// --------------------------------------------------------------------------- +// Request translation: Anthropic → OpenAI +// --------------------------------------------------------------------------- + +export const decodeOpaqueServerToolContent = (value: unknown): unknown => { + if (typeof value !== 'string' || !value) { + return null; + } + + try { + const binary = atob(value); + const bytes = Uint8Array.from(binary, (character) => + character.charCodeAt(0), + ); + + return JSON.parse(new TextDecoder().decode(bytes)) as unknown; + } catch { + return null; + } +}; + +export const formatAnthropicServerToolResult = ( + block: AnthropicContentBlock, +): string => { + if (block.type === 'web_search_tool_result' && Array.isArray(block.content)) { + return block.content + .map((value, index) => { + const item = + value && typeof value === 'object' + ? (value as Record) + : {}; + const decoded = decodeOpaqueServerToolContent(item.encrypted_content); + const source = + decoded && typeof decoded === 'object' + ? (decoded as Record) + : item; + const title = String(source.title ?? item.title ?? '').trim(); + const url = String(source.url ?? item.url ?? '').trim(); + const text = String( + source.content ?? source.snippet ?? source.text ?? '', + ).trim(); + + return [ + `${index + 1}. ${title || url || 'Search result'}`, + ...(url ? [`URL: ${url}`] : []), + ...(text ? [text] : []), + ].join('\n'); + }) + .join('\n\n'); + } + + if ( + block.type === 'web_fetch_tool_result' && + block.content && + typeof block.content === 'object' + ) { + const result = block.content as Record; + const document = + result.content && typeof result.content === 'object' + ? (result.content as Record) + : null; + const source = + document?.source && typeof document.source === 'object' + ? (document.source as Record) + : null; + const url = typeof result.url === 'string' ? result.url : ''; + const text = typeof source?.data === 'string' ? source.data : ''; + + return [url, text].filter(Boolean).join('\n\n'); + } + + // Nested images are emitted as real image parts by + // `collectAnthropicNestedImages`, so they are excluded here to keep their + // base64 payload out of the text. + if (Array.isArray(block.content)) { + return stringifyContent( + block.content.filter((value) => { + return !( + value && + typeof value === 'object' && + (value as AnthropicContentBlock).type === 'image' + ); + }), + ); + } + + return typeof block.content === 'string' + ? block.content + : stringifyContent(block.content); +}; + +/** + * Images nested inside a `tool_result` content array, e.g. a screenshot a tool + * returned. The outer block is handled by the `tool_result` branch, whose + * formatter stringifies nested content — so without extracting them here the + * model would receive the base64 payload as text. + */ +export const collectAnthropicNestedImages = ( + block: AnthropicContentBlock, +): ChatImageBlock[] => { + if (!Array.isArray(block.content)) { + return []; + } + + return block.content.flatMap((value): ChatImageBlock[] => { + if (!value || typeof value !== 'object') { + return []; + } + + const nested = value as AnthropicContentBlock; + + if (nested.type !== 'image') { + return []; + } + + const imageUrl = buildChatImageUrl(nested.source); + + return imageUrl + ? [{ type: 'image_url', image_url: { url: imageUrl } }] + : []; + }); +}; + +export const mapAnthropicContentToChat = ( + content: string | AnthropicContentBlock[], + role: 'user' | 'assistant', +): ChatMessage[] => { + if (typeof content === 'string') { + return [{ role, content }]; + } + + const parts: ChatContentPart[] = []; + const toolCalls: Array<{ + id: string; + type: string; + function: { + name: string; + arguments: string; + }; + }> = []; + const toolResults: ChatMessage[] = []; + const messages: ChatMessage[] = []; + /** + * Reasoning recovered from thinking blocks in this assistant message. + * + * Attached to the message the blocks belong to rather than sent on its own: + * a bare reasoning entry is not a valid chat message, and the upstream needs + * the reasoning alongside the text and tool calls it produced. + */ + let pendingReasoning = ''; + const flushAssistantMessage = (): void => { + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; + + if (!toolCalls.length && !hasContent && !pendingReasoning) { + return; + } + + messages.push({ + role: 'assistant', + content: hasContent ? content : null, + ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}), + // `reasoning` is the field the CodeBuddy chat upstream round-trips. It + // is not part of the OpenAI schema, but the upstream accepts it and + // ignoring an unknown field costs nothing if it ever stops doing so. + ...(pendingReasoning ? { reasoning: pendingReasoning } : {}), + }); + parts.length = 0; + toolCalls.length = 0; + pendingReasoning = ''; + }; + + for (const block of content) { + if (block.type === 'text') { + const text = block.text ?? ''; + + parts.push( + block.cache_control + ? { type: 'text', text, cache_control: block.cache_control } + : text, + ); + } else if (block.type === 'tool_use' || block.type === 'server_tool_use') { + toolCalls.push({ + id: block.id ?? createAnthropicId('toolu'), + type: 'function', + function: { + name: block.name ?? 'unknown', + arguments: JSON.stringify(block.input ?? {}), + }, + }); + } else if ( + block.type === 'tool_result' || + block.type === 'web_search_tool_result' || + block.type === 'web_fetch_tool_result' + ) { + const nestedImages = collectAnthropicNestedImages(block); + + const resultMessage: ChatMessage = { + role: 'tool', + content: nestedImages.length + ? mapContentPartsToChat([ + formatAnthropicServerToolResult(block), + ...nestedImages, + ]) + : formatAnthropicServerToolResult(block), + tool_call_id: block.tool_use_id ?? '', + }; + + if (role === 'assistant') { + flushAssistantMessage(); + messages.push(resultMessage); + } else { + toolResults.push(resultMessage); + } + } else if ( + block.type === 'thinking' || + block.type === 'redacted_thinking' + ) { + // Replaying prior-turn reasoning is required inside a tool-use turn and + // harmless elsewhere, so recover it instead of dropping it. + // + // The `thinking` field carries the reasoning. A `signature` is only ever + // read when it is one we minted on the Responses path; a genuine + // Anthropic signature is ciphertext, and forwarding it upstream would put + // gibberish where reasoning belongs. + // + // `redacted_thinking` has no readable text at all, only `data`, but must + // still be matched here: without this branch it fell through to + // `stringifyContent` and the model received a JSON dump of the opaque + // payload as if it were user prose. + const reasoning = block.thinking ?? ''; + + if (reasoning) { + pendingReasoning = pendingReasoning + ? `${pendingReasoning}${reasoning}` + : reasoning; + } + } else if (block.type === 'image' && block.source) { + // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url', + // media_type, data | url } }`. Emit a real image block so the upstream + // model sees the image; without this branch the block fell through to + // `stringifyContent` and the model received a JSON dump of the base64 + // payload as text. An `image` block with no `source` is not a real + // Anthropic image, so it keeps the generic stringified handling. + const imageUrl = buildChatImageUrl(block.source); + + parts.push( + imageUrl + ? { + type: 'image_url', + image_url: { url: imageUrl }, + // Preserve an explicit cache breakpoint, matching how text + // blocks carry `cache_control` through. Without this the + // requested breakpoint is dropped and `applyPromptCacheControl` + // falls back to its own automatic placement. + ...(block.cache_control + ? { cache_control: block.cache_control } + : {}), + } + : stringifyContent(block), + ); + } else { + parts.push(stringifyContent(block)); + } + } + + if (role === 'user') { + messages.push(...toolResults); + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; + if (hasContent) { + messages.push({ role: 'user', content }); + } + } else { + flushAssistantMessage(); + } + + return messages; +}; + +export const mapAnthropicMessagesToChat = ( + messages: AnthropicMessage[], +): ChatMessage[] => { + const result: ChatMessage[] = []; + + for (const msg of messages) { + const mapped = mapAnthropicContentToChat(msg.content, msg.role); + + if (mapped.length === 0) { + continue; + } + + for (const item of mapped) { + result.push({ + ...item, + }); + } + } + + return result; +}; + +export const mapAnthropicToolsToChat = ( + tools: AnthropicTool[] | undefined, +): unknown[] | undefined => { + if (!tools?.length) { + return undefined; + } + + return tools.map((tool) => { + const mapped = { + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: tool.input_schema, + }, + }; + const normalizedType = normalizeToolName(tool.type ?? ''); + const serverDeclared = [ + WEB_SEARCH_TOOL_TYPE_PREFIX, + WEB_FETCH_TOOL_TYPE_PREFIX, + ].some((prefix) => normalizedType.startsWith(normalizeToolName(prefix))); + + return serverDeclared ? markServerTool(mapped) : mapped; + }); +}; + +export const shouldBridgeAnthropicServerTools = async ( + tools: AnthropicTool[] | undefined, +): Promise => { + const names = new Set( + (tools ?? []).map((tool) => normalizeToolName(tool.name)), + ); + const [searchEnabled, fetchEnabled] = await Promise.all([ + names.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) + ? isWebSearchEnabled() + : false, + names.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) + ? isWebFetchEnabled() + : false, + ]); + + return searchEnabled || fetchEnabled; +}; + +export const mapAnthropicToolChoiceToChat = (toolChoice: unknown): unknown => { + if (!toolChoice || typeof toolChoice !== 'object') { + return toolChoice; + } + + const tc = toolChoice as { type?: string; name?: string }; + + if (tc.type === 'auto') { + return 'auto'; + } + + if (tc.type === 'any') { + return 'required'; + } + + if (tc.type === 'tool' && tc.name) { + return { + type: 'function', + function: { name: tc.name }, + }; + } + + if (tc.type === 'none') { + return 'none'; + } + + return toolChoice; +}; + +export const buildChatRequestBody = async ( + body: AnthropicMessagesRequestBody, +): Promise> => { + const systemText = extractSystemText(body.system); + const chatMessages = mapAnthropicMessagesToChat(body.messages ?? []); + + const messages: ChatMessage[] = []; + const disableParallelToolUse = + body.tool_choice && typeof body.tool_choice === 'object' + ? (body.tool_choice as { disable_parallel_tool_use?: unknown }) + .disable_parallel_tool_use + : undefined; + + if (systemText) { + messages.push({ role: 'system', content: systemText }); + } + + messages.push(...chatMessages); + + const result: Record = { + model: + typeof body.model === 'string' && body.model.trim() + ? body.model + : await getDefaultModel('claude-sonnet-4.6'), + messages, + stream: body.stream ?? false, + max_tokens: body.max_tokens, + temperature: body.temperature, + top_p: body.top_p, + stop: body.stop_sequences, + tools: mapAnthropicToolsToChat(body.tools), + tool_choice: mapAnthropicToolChoiceToChat(body.tool_choice), + parallel_tool_calls: + typeof disableParallelToolUse === 'boolean' + ? !disableParallelToolUse + : undefined, + }; + + // Pass through thinking/reasoning config so upstream models that support + // extended thinking can honor it. + if (body.thinking) { + result.thinking = body.thinking; + } + + return result; +}; diff --git a/lib/server/proxy/anthropic/response.ts b/lib/server/proxy/anthropic/response.ts new file mode 100644 index 0000000..d4c57a3 --- /dev/null +++ b/lib/server/proxy/anthropic/response.ts @@ -0,0 +1,255 @@ +import { createAnthropicId } from './content'; +import type { + AnthropicContentBlock, + OpenAIChatResponse, + OpenAIUsage, +} from './types'; +import type { ServerToolExecution, ServerToolTurn } from '../web-search-loop'; + +// --------------------------------------------------------------------------- +// Response translation: OpenAI → Anthropic (non-streaming) +// --------------------------------------------------------------------------- + +export const mapOpenAIUsageToAnthropic = ( + usage: OpenAIUsage | undefined, + serverToolExecutions: ServerToolExecution[] = [], +): Record => { + const cacheCreationTokens = + usage?.prompt_tokens_details?.cache_creation_tokens ?? 0; + const cacheReadTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; + // prompt_tokens is the total prompt count including cached tokens. + // Anthropic reports cached/created tokens separately, so input_tokens + // must be the non-cache remainder to avoid double-counting. + const inputTokens = Math.max( + 0, + (usage?.prompt_tokens ?? 0) - cacheCreationTokens - cacheReadTokens, + ); + const outputTokens = usage?.completion_tokens ?? 0; + + const mapped: Record> = { + input_tokens: inputTokens, + output_tokens: outputTokens, + cache_creation_input_tokens: cacheCreationTokens, + cache_read_input_tokens: cacheReadTokens, + }; + + if (serverToolExecutions.length) { + mapped.server_tool_use = { + web_search_requests: serverToolExecutions.filter( + (execution) => execution.type === 'web_search', + ).length, + web_fetch_requests: serverToolExecutions.filter( + (execution) => execution.type === 'web_fetch', + ).length, + }; + } + + return mapped; +}; + +export const encodeOpaqueServerToolContent = (value: unknown): string => { + const bytes = new TextEncoder().encode(JSON.stringify(value)); + let binary = ''; + + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + + return btoa(binary); +}; + +export const buildAnthropicServerToolBlocks = ( + execution: ServerToolExecution, +): AnthropicContentBlock[] => { + const id = createAnthropicId('srvtoolu'); + const result = + execution.type === 'web_search' + ? { + type: 'web_search_tool_result', + tool_use_id: id, + content: execution.result.results.map((item) => ({ + type: 'web_search_result', + url: item.url ?? '', + title: item.title ?? '', + encrypted_content: encodeOpaqueServerToolContent(item), + })), + } + : { + type: 'web_fetch_tool_result', + tool_use_id: id, + content: { + type: 'web_fetch_result', + url: execution.result.url ?? execution.input.url, + content: { + type: 'document', + source: { + type: 'text', + media_type: 'text/plain', + data: execution.result.content, + }, + }, + }, + }; + + return [ + { + type: 'server_tool_use', + id, + name: execution.type, + input: execution.input, + }, + result, + ]; +}; + +export const buildAllAnthropicServerToolBlocks = ( + executions: ServerToolExecution[], +): AnthropicContentBlock[] => + executions.flatMap(buildAnthropicServerToolBlocks); + +/** + * We do not mint a `signature` on this path. It would have to duplicate the + * `thinking` text to be replayable, which puts the reasoning on the wire twice + * for callers that count it — and the block already replays fine: Anthropic + * clients echo `thinking` back, which is what inbound handling reads. + */ +export const buildThinkingBlock = ( + thinking: string, +): AnthropicContentBlock => ({ + type: 'thinking', + thinking, +}); + +/** + * Lays a server-tool turn out the way Anthropic does: each hop contributes its + * own thinking and text, followed by the tool blocks that hop triggered. + * + * `turns` carries the per-hop grouping the OpenAI-shaped payload cannot. Under + * that protocol a multi-hop turn collapses into one `content` string and one + * `reasoning_content` string, which loses where one hop's reasoning ends and the + * next begins — so the grouping has to be recovered before it is joined, which + * is why the loop emits it alongside the strings rather than this file + * reconstructing it. + * + * Anthropic's own server tools run multiple hops inside one assistant message, + * and a client replaying that message expects `[thinking] [text] [tool_use] + * [tool_result] [thinking] [text]`. Gathering the blocks by kind instead — every + * tool ahead of all the prose — puts each search before the reasoning that asked + * for it and merges hops that were never contiguous. + */ +export const buildAnthropicTurnBlocks = ( + turns: ServerToolTurn[], +): AnthropicContentBlock[] => { + const blocks: AnthropicContentBlock[] = []; + + turns.forEach((turn) => { + if (turn.reasoning) { + blocks.push(buildThinkingBlock(turn.reasoning)); + } + + if (turn.text) { + blocks.push({ type: 'text', text: turn.text }); + } + + blocks.push(...buildAllAnthropicServerToolBlocks(turn.executions)); + }); + + return blocks; +}; + +export const mapOpenAIResponseToAnthropic = ( + openaiResponse: OpenAIChatResponse, + model: string, + serverToolExecutions: ServerToolExecution[] = [], + turns?: ServerToolTurn[], +): Record => { + const choice = openaiResponse.choices?.[0]; + const message = choice?.message; + + // Thinking / reasoning content + const reasoningText = message?.reasoning_content ?? message?.reasoning ?? ''; + + // Text content + const textContent = + typeof message?.content === 'string' ? message.content : ''; + + // With per-hop grouping the turns already hold every block in order, prose + // included. Without it — no server tool ran, or a path that never grouped the + // hops — fall back to Anthropic's own order: thinking and text first, then + // the server-tool blocks they led to. + const contentBlocks: AnthropicContentBlock[] = turns + ? buildAnthropicTurnBlocks(turns) + : []; + + if (!turns) { + if (reasoningText) { + contentBlocks.push(buildThinkingBlock(reasoningText)); + } + + if (textContent) { + contentBlocks.push({ type: 'text', text: textContent }); + } + + contentBlocks.push( + ...buildAllAnthropicServerToolBlocks(serverToolExecutions), + ); + } + + // Tool calls + const toolCalls = message?.tool_calls ?? []; + + for (const call of toolCalls) { + let input: unknown = {}; + + try { + input = JSON.parse(call.function?.arguments ?? '{}'); + } catch { + input = {}; + } + + contentBlocks.push({ + type: 'tool_use', + id: call.id ?? createAnthropicId('toolu'), + name: call.function?.name ?? 'unknown', + input, + }); + } + + const stopReason = mapFinishReasonToAnthropic( + choice?.finish_reason, + toolCalls.length > 0, + ); + + return { + id: openaiResponse.id ?? createAnthropicId('msg'), + type: 'message', + role: 'assistant', + model, + content: contentBlocks, + stop_reason: stopReason, + stop_sequence: null, + usage: mapOpenAIUsageToAnthropic( + openaiResponse.usage, + serverToolExecutions, + ), + }; +}; + +export const mapFinishReasonToAnthropic = ( + finishReason: string | null | undefined, + hasToolCalls: boolean, +): string => { + if (hasToolCalls || finishReason === 'tool_calls') { + return 'tool_use'; + } + + if (finishReason === 'length') { + return 'max_tokens'; + } + + if (finishReason === 'stop' || !finishReason) { + return 'end_turn'; + } + + return 'end_turn'; +}; diff --git a/lib/server/proxy/anthropic/stream.ts b/lib/server/proxy/anthropic/stream.ts new file mode 100644 index 0000000..7e729fa --- /dev/null +++ b/lib/server/proxy/anthropic/stream.ts @@ -0,0 +1,633 @@ +import type { NextRequest } from 'next/server'; + +import type { DebugTrace } from '../../domain/debug'; +import { createSseResponse } from '../../shared/sse'; +import { + anthropicStreamErrorChunks, + createStreamCloser, + toUpstreamTimeoutMessage, +} from '../../shared/upstream-timeout'; +import { proxyChatCompletions, type ChatRequestBody } from '../codebuddy'; +import { createAnthropicId } from './content'; +import { anthropicErrorType, getUpstreamErrorMessage } from './errors'; +import { + buildAnthropicServerToolBlocks, + mapFinishReasonToAnthropic, + mapOpenAIUsageToAnthropic, +} from './response'; +import { MAX_STREAM_FRAME_LENGTH } from './types'; +import type { + OpenAIStreamChunk, + OpenAIStreamError, + OpenAIUsage, + StreamingToolUseState, +} from './types'; +import { + getServerToolExecutions, + getServerToolStreamEvent, + type ServerToolExecution, +} from '../web-search-loop'; + +// --------------------------------------------------------------------------- +// Response translation: OpenAI SSE → Anthropic SSE (streaming) +// --------------------------------------------------------------------------- + +export const mapOpenAIStreamToAnthropicSSE = ( + upstreamResponse: Response, + model: string, + options?: { + emitMessageStart?: boolean; + initialContentBlockCount?: number; + messageId?: string; + serverToolExecutions?: ServerToolExecution[]; + }, +): Response => { + if (!upstreamResponse.body) { + return createSseResponse(null, { status: upstreamResponse.status }); + } + + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + + const messageId = options?.messageId ?? createAnthropicId('msg'); + const serverToolExecutions = + options?.serverToolExecutions ?? getServerToolExecutions(upstreamResponse); + const serverToolUseIds = new Map(); + const toolUseStates = new Map(); + let nextToolIndex = 0; + let started = options?.emitMessageStart === false; + let thinkingStarted = false; + let thinkingBlockIndex = -1; + let textStarted = false; + let textBlockIndex = -1; + // Tracks how many content blocks (thinking + text) have been opened + // so tool_use blocks get correct sequential indices even after the + // prior blocks are closed mid-stream. + let contentBlockCount = options?.initialContentBlockCount ?? 0; + let finishReason: string | null = null; + let hasToolCalls = false; + let usage: OpenAIUsage | undefined; + + const enqueueEvent = (event: Record): void => { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ); + }; + + let controller: ReadableStreamDefaultController; + + // Close any open text/thinking block before starting a tool_use block. + // Anthropic streaming requires each block to be stopped before the next. + const closeOpenTextBlocks = (): void => { + if (thinkingStarted) { + // Anthropic emits the signature last, just before the block closes — but + // we do not send one here: the reasoning already went out as + // `thinking_delta`s, and duplicating it into a signature would put the + // text on the wire twice for callers that count it. See + // `buildThinkingBlock`. + enqueueEvent({ + type: 'content_block_stop', + index: thinkingBlockIndex, + }); + thinkingStarted = false; + } + + if (textStarted) { + enqueueEvent({ + type: 'content_block_stop', + index: textBlockIndex, + }); + textStarted = false; + } + }; + + const processChunk = (chunk: OpenAIStreamChunk): void => { + if (!started) { + started = true; + enqueueEvent({ + type: 'message_start', + message: { + id: messageId, + type: 'message', + role: 'assistant', + content: [], + model, + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: chunk.usage?.prompt_tokens ?? 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + }); + } + + if (chunk.usage) { + usage = chunk.usage; + } + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (!delta) { + return; + } + + // Reasoning / thinking content + const reasoningText = delta.reasoning_content ?? delta.reasoning ?? ''; + + if (reasoningText) { + if (!thinkingStarted) { + thinkingBlockIndex = contentBlockCount; + thinkingStarted = true; + enqueueEvent({ + type: 'content_block_start', + index: thinkingBlockIndex, + content_block: { + type: 'thinking', + thinking: '', + }, + }); + contentBlockCount++; + } + + enqueueEvent({ + type: 'content_block_delta', + index: thinkingBlockIndex, + delta: { + type: 'thinking_delta', + thinking: reasoningText, + }, + }); + } + + // Text content + if (delta.content) { + if (!textStarted) { + // Close the thinking block before starting text so Anthropic + // stream consumers see properly ordered, non-overlapping blocks. + closeOpenTextBlocks(); + + textBlockIndex = contentBlockCount; + textStarted = true; + enqueueEvent({ + type: 'content_block_start', + index: textBlockIndex, + content_block: { + type: 'text', + text: '', + }, + }); + contentBlockCount++; + } + + enqueueEvent({ + type: 'content_block_delta', + index: textBlockIndex, + delta: { + type: 'text_delta', + text: delta.content, + }, + }); + } + + // Tool calls + if (delta.tool_calls?.length) { + hasToolCalls = true; + + // Anthropic streaming requires each content block to be closed + // before the next one starts. If we already opened a text or + // thinking block, close it now so the tool_use block is well-formed. + closeOpenTextBlocks(); + + for (const call of delta.tool_calls) { + const callId = call.id ?? `toolu_${nextToolIndex}`; + const key = callId; + + if (!toolUseStates.has(key)) { + const blockIndex = contentBlockCount + nextToolIndex; + + toolUseStates.set(key, { + id: callId, + name: '', + input: '', + index: blockIndex, + started: false, + blockEmitted: false, + }); + nextToolIndex++; + } + + const state = toolUseStates.get(key)!; + + // Accumulate name fragments (upstream may stream the function + // name across multiple deltas, e.g. "look" + "up"). + if (call.function?.name) { + state.name += call.function.name; + } + + // Emit content_block_start lazily — once we have a name and at + // least one arguments fragment, so the block header carries the + // full tool name instead of a partial fragment. + if (!state.blockEmitted && state.name && call.function?.arguments) { + state.blockEmitted = true; + enqueueEvent({ + type: 'content_block_start', + index: state.index, + content_block: { + type: 'tool_use', + id: state.id, + name: state.name, + input: {}, + }, + }); + } + + if (call.function?.arguments) { + state.input += call.function.arguments; + enqueueEvent({ + type: 'content_block_delta', + index: state.index, + delta: { + type: 'input_json_delta', + partial_json: call.function.arguments, + }, + }); + } + } + } + + if (choice?.finish_reason) { + finishReason = choice.finish_reason; + } + }; + + const finalize = (): void => { + // Close any remaining open text/thinking blocks. + closeOpenTextBlocks(); + + // Close tool use blocks + for (const [, state] of toolUseStates) { + // If the block start was never emitted (e.g. name-only deltas + // with no arguments), emit it now so the block is well-formed. + if (!state.blockEmitted) { + state.blockEmitted = true; + enqueueEvent({ + type: 'content_block_start', + index: state.index, + content_block: { + type: 'tool_use', + id: state.id, + name: state.name || 'unknown', + input: {}, + }, + }); + } + + enqueueEvent({ + type: 'content_block_stop', + index: state.index, + }); + } + + const stopReason = mapFinishReasonToAnthropic(finishReason, hasToolCalls); + + enqueueEvent({ + type: 'message_delta', + delta: { + stop_reason: stopReason, + stop_sequence: null, + }, + usage: mapOpenAIUsageToAnthropic(usage, serverToolExecutions), + }); + + enqueueEvent({ + type: 'message_stop', + }); + }; + + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + let streamRejected = false; + const closer = createStreamCloser(); + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + const stream = new ReadableStream({ + start: (ctrl) => { + controller = ctrl; + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + const rejectStream = ( + message = 'Upstream SSE frame exceeds the maximum size', + status?: number, + ): void => { + streamRejected = true; + enqueueEvent({ + type: 'error', + error: { + // An upstream status names the failure precisely, so it decides + // the type: 429 has to arrive as `rate_limit_error` or a client + // that retries on that type alone stops retrying an exhausted + // quota. Without one, fall back to the message: an oversized frame + // is a malformed stream (`invalid_request_error`), while an + // upstream deadline is the server failing (`api_error`, the type + // clients treat as retryable). + type: + typeof status === 'number' + ? anthropicErrorType(status) + : message.includes('did not produce output') + ? 'api_error' + : 'invalid_request_error', + message, + }, + }); + }; + + const flushFrames = (frames: string[]): void => { + for (const frame of frames) { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + rejectStream(); + return; + } + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + continue; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + try { + const chunk = JSON.parse(raw) as OpenAIStreamChunk; + const serverToolEvent = getServerToolStreamEvent(chunk); + + if (serverToolEvent?.phase === 'call') { + closeOpenTextBlocks(); + const index = contentBlockCount++; + const toolUseId = createAnthropicId('srvtoolu'); + serverToolUseIds.set(serverToolEvent.invocation.id, toolUseId); + enqueueEvent({ + type: 'content_block_start', + index, + content_block: { + type: 'server_tool_use', + id: toolUseId, + name: serverToolEvent.invocation.type, + input: {}, + }, + }); + enqueueEvent({ + type: 'content_block_delta', + index, + delta: { + type: 'input_json_delta', + partial_json: JSON.stringify( + serverToolEvent.invocation.input, + ), + }, + }); + enqueueEvent({ type: 'content_block_stop', index }); + continue; + } + + if (serverToolEvent?.phase === 'result') { + closeOpenTextBlocks(); + serverToolExecutions.push(serverToolEvent.execution); + const index = contentBlockCount++; + const resultBlock = buildAnthropicServerToolBlocks( + serverToolEvent.execution, + )[1]; + enqueueEvent({ + type: 'content_block_start', + index, + content_block: { + ...resultBlock, + tool_use_id: serverToolUseIds.get( + serverToolEvent.execution.id, + ), + }, + }); + enqueueEvent({ type: 'content_block_stop', index }); + continue; + } + + const upstreamError = chunk as OpenAIStreamError; + if (upstreamError.error?.message) { + rejectStream( + upstreamError.error.message, + upstreamError.error.status, + ); + return; + } + processChunk(chunk); + } catch { + // Skip unparseable frames + } + } + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + if (buffer.trim()) { + flushFrames([buffer]); + } + if (!streamRejected) { + finalize(); + } + releaseReader(); + controller.close(); + return; + } + + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + rejectStream(); + } else { + flushFrames(frames); + } + if (streamRejected) { + try { + await reader!.cancel(); + } finally { + releaseReader(); + controller.close(); + } + return; + } + } + }; + + void pump().catch((error) => { + if (cancelled) return; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage === null) { + closer.mark(); + controller.error(error); + return; + } + + void reader?.cancel().then( + () => undefined, + () => undefined, + ); + releaseReader(); + closer.fail(controller, anthropicStreamErrorChunks(timeoutMessage)); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + releaseReader(); + } + }, + }); + + return createSseResponse(stream, { status: 200 }); +}; + +export const createAnthropicServerToolEventStream = ( + request: NextRequest, + chatBody: Record, + model: string, + debugTrace?: DebugTrace, +): Response => { + const encoder = new TextEncoder(); + const messageId = createAnthropicId('msg'); + let activeReader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + + const stream = new ReadableStream({ + start: (controller) => { + const enqueueEvent = (event: Record): void => { + if (cancelled) return; + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ); + }; + + enqueueEvent({ + type: 'message_start', + message: { + id: messageId, + type: 'message', + role: 'assistant', + content: [], + model, + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + }); + + const run = async (): Promise => { + const upstreamResponse = await proxyChatCompletions( + request, + chatBody as ChatRequestBody, + undefined, + debugTrace, + '/v1/messages', + // The findings reach the client as `web_search_tool_result` blocks, + // so the loop must not also fold them into the assistant text. + { emitStreamEvents: true, findingsAsStructuredBlocks: true }, + ); + + if (cancelled) { + await upstreamResponse.body?.cancel(); + return; + } + + if (!upstreamResponse.ok || !upstreamResponse.body) { + // A rate limit has to arrive as `rate_limit_error`, or a client that + // retries on that type alone will treat an exhausted quota as a + // generic failure and stop retrying — so the upstream status drives + // the event type even though the envelope is already streaming and + // the HTTP status cannot be changed. + const message = upstreamResponse.ok + ? 'Upstream request failed' + : await getUpstreamErrorMessage(upstreamResponse).catch( + () => 'Upstream request failed', + ); + + enqueueEvent({ + type: 'error', + error: { + type: anthropicErrorType(upstreamResponse.status), + message, + }, + }); + controller.close(); + return; + } + + const mappedResponse = mapOpenAIStreamToAnthropicSSE( + upstreamResponse, + model, + { + emitMessageStart: false, + initialContentBlockCount: 0, + messageId, + serverToolExecutions: [], + }, + ); + const reader = mappedResponse.body!.getReader(); + activeReader = reader; + + while (true) { + const { done, value } = await reader.read(); + if (cancelled) return; + if (done) break; + controller.enqueue(value); + } + + reader.releaseLock(); + activeReader = null; + controller.close(); + }; + + void run().catch((error) => { + if (!cancelled) controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + await activeReader?.cancel(reason); + activeReader?.releaseLock(); + activeReader = null; + }, + }); + + return createSseResponse(stream); +}; diff --git a/lib/server/proxy/anthropic/types.ts b/lib/server/proxy/anthropic/types.ts new file mode 100644 index 0000000..d1ebdee --- /dev/null +++ b/lib/server/proxy/anthropic/types.ts @@ -0,0 +1,184 @@ +// --------------------------------------------------------------------------- +// Anthropic Messages API types +// --------------------------------------------------------------------------- + +export const MAX_STREAM_FRAME_LENGTH = 1_000_000; + +export interface AnthropicImageSource { + type?: string; + media_type?: string; + data?: string; + url?: string; +} + +export interface AnthropicContentBlock { + type: string; + text?: string; + cache_control?: { type?: string }; + id?: string; + name?: string; + input?: unknown; + thinking?: string; + /** + * Accepted on inbound blocks but never sent by us — see + * `buildThinkingBlock`. Anthropic's signatures hold an encrypted copy of the + * reasoning; a client may replay one from a session it started elsewhere, and + * we skip those rather than forward ciphertext as if it were text. + */ + signature?: string; + /** Present on `redacted_thinking` blocks, which carry no readable text. */ + data?: string; + tool_use_id?: string; + content?: unknown; + source?: AnthropicImageSource; +} + +export interface AnthropicMessage { + role: 'user' | 'assistant'; + content: string | AnthropicContentBlock[]; +} + +export interface AnthropicTool { + name: string; + description?: string; + input_schema: Record; + type?: string; +} + +export interface AnthropicThinkingConfig { + type?: string; + budget_tokens?: number; +} + +export interface AnthropicMessagesRequestBody { + model?: string; + messages?: AnthropicMessage[]; + system?: string | AnthropicContentBlock[]; + max_tokens?: number; + temperature?: number; + top_p?: number; + top_k?: number; + stop_sequences?: string[]; + stream?: boolean; + tools?: AnthropicTool[]; + tool_choice?: unknown; + thinking?: AnthropicThinkingConfig; + metadata?: Record; +} + +// --------------------------------------------------------------------------- +// OpenAI response types (mirrors of codebuddy.ts internals) +// --------------------------------------------------------------------------- + +export interface OpenAIToolCall { + index?: number; + id?: string; + type?: string; + function?: { + arguments?: string; + name?: string; + }; +} + +export interface OpenAIChatMessage { + role?: string; + content?: unknown; + tool_calls?: OpenAIToolCall[]; + reasoning_content?: string; + reasoning?: string; +} + +export interface OpenAIChatChoice { + index?: number; + message?: OpenAIChatMessage; + delta?: OpenAIChatMessage; + finish_reason?: string | null; +} + +export interface OpenAIStreamError { + error?: { message?: string; status?: number }; +} + +export interface OpenAIUsage { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + prompt_tokens_details?: { + cached_tokens?: number; + cache_creation_tokens?: number; + }; + completion_tokens_details?: { + reasoning_tokens?: number; + }; +} + +export interface OpenAIChatResponse { + id?: string; + model?: string; + choices?: OpenAIChatChoice[]; + usage?: OpenAIUsage; +} + +export interface OpenAIStreamChunk { + id?: string; + model?: string; + choices?: OpenAIChatChoice[]; + usage?: OpenAIUsage; +} + +export interface ChatTextBlock { + cache_control?: { type?: string }; + text: string; + type: 'text'; +} + +/** + * An image part in the OpenAI Chat shape. Emitted in this shape rather than a + * native Anthropic one because the request is translated to Chat before it + * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the + * `responses` upstream converts it to `input_image`. + */ +export interface ChatImageBlock { + cache_control?: { type?: string }; + image_url: { url: string }; + type: 'image_url'; +} + +export type ChatContentPart = string | ChatTextBlock | ChatImageBlock; + +export type ChatContent = string | Array; + +/** + * Text-only content, used where images are not representable — the system + * prompt and the intermediate text-part buffer. + */ +export type ChatTextContent = string | ChatTextBlock[]; + +export interface ChatMessage { + role: string; + content: ChatContent | null; + tool_calls?: Array<{ + id: string; + type: string; + function: { + name: string; + arguments: string; + }; + }>; + tool_call_id?: string; + /** + * Prior-turn reasoning for this assistant message. Not part of the OpenAI + * schema; the CodeBuddy chat upstream round-trips it, and a provider that + * does not know the field ignores it. + */ + reasoning?: string; +} + +export interface StreamingToolUseState { + id: string; + name: string; + input: string; + index: number; + started: boolean; + blockEmitted: boolean; +} diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index e8f7a12..02f3476 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -1,2917 +1,66 @@ import type { NextRequest } from 'next/server'; -import { resolveRequestAccessKey } from './auth'; import { - getApiFirstDeltaTimeoutMs, - getCodeBuddyApiEndpoint, - getDefaultModel, -} from '../domain/config'; -import { - type CredentialData, - type CredentialRecord, - findEligibleCredentialRecordByFilename, - findCredentialRecordByFilename, - getCredentialSupportedModels, - getCredentialProxySettings, - listEligibleCredentialRecords, - resolveCredentialForRequest, -} from '../domain/credentials'; -import { - enqueueUpstreamResponseSnapshot, - setDebugTraceCredential, - setDebugTraceError, - setDebugUpstreamRequest, - type DebugTrace, -} from '../domain/debug'; -import { createErrorResponse, getRequestHeaderMap } from '../shared/http'; -import { - resolveHyChatThinking, - resolveHyResponsesReasoning, -} from '../shared/hy-thought-depth'; -import { withCodeBuddyToken } from '../search/token'; -import { - normalizeToolName, - WEB_FETCH_TOOL_NAME, - WEB_SEARCH_TOOL_NAME, -} from '../search/tool'; -import { - attachServerToolExecutions, - attachServerToolTurns, - type ChatCompletionPayload, - executeWebSearchLoop, - type ServerToolCallbacks, - synthesizeChatCompletionStream, -} from './web-search-loop'; -import { - chatStreamErrorChunks, - createStreamCloser, - fetchWithDeadline, - responsesStreamErrorChunks, - readTimeoutFrame, - toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { recordUsageEvent, type UsageSnapshot } from '../domain/usage'; - -interface OpenAIMessage { - role?: string; - content?: unknown; - tool_calls?: unknown[]; - tool_call_id?: string; - /** - * Prior-turn reasoning for an assistant message. - * - * Not an OpenAI field. The CodeBuddy chat upstream accepts it on assistant - * messages and uses it to carry reasoning across turns — the same slot - * CodeBuddy's own client populates when it replays a response. It is named - * `reasoning` rather than `reasoning_content` because that one is the - * upstream's *response* field; this is the request-side counterpart. - */ - reasoning?: string; -} - -interface CacheableTextBlock { - cache_control?: { type: 'ephemeral' }; - text: string; - type: 'text'; -} - -const MIN_AUTO_CACHE_TEXT_LENGTH = 1024; -const MAX_STREAM_FRAME_LENGTH = 1_000_000; -const CODEBUDDY_CLI_VERSION = '2.137.1'; -const CODEBUDDY_USER_AGENT = `CLI/${CODEBUDDY_CLI_VERSION} CodeBuddy/${CODEBUDDY_CLI_VERSION}`; - -export interface ChatRequestBody { - model?: string; - messages?: OpenAIMessage[]; - stream?: boolean; - stream_options?: { - include_usage?: boolean; - }; - temperature?: number; - max_tokens?: number; - max_completion_tokens?: number; - response_format?: unknown; - top_p?: number; - frequency_penalty?: number; - presence_penalty?: number; - stop?: string | string[]; - tools?: unknown[]; - tool_choice?: unknown; - parallel_tool_calls?: boolean; - thinking?: Record; - reasoning_effort?: string; -} - -interface ChatStreamDelta { - content?: string; - role?: string; - reasoning_content?: string; - reasoning?: string; - tool_calls?: Array<{ - index?: number; - id?: string; - type?: string; - function?: { - arguments?: string; - name?: string; - }; - }>; -} - -interface ChatStreamChunk { - id?: string; - object?: string; - created?: number; - model?: string; - usage?: unknown; - choices?: Array<{ - delta?: ChatStreamDelta; - finish_reason?: string | null; - index?: number; - }>; -} - -type ToolCallChunk = NonNullable[number]; - -interface ToolCallMapping { - id: string; - index: number; -} - -interface ToolCallNormalizationState { - mappings: Map; - nextIndex: number; -} - -interface ResolvedAuth { - type: 'bearer'; - bearerToken: string; - userId: string; - credentialData: Record; -} - -export interface ProxyContext { - accessKeyId: string | null; - accessKeyName: string | null; - auth: ResolvedAuth; - credentialFilename: string | null; - preferences: { - firstMessageRoleToSystem: boolean; - firstSystemMessageRoleToUser: boolean; - upstreamProtocol: 'chat' | 'responses'; - }; -} - -export interface DiscoveredModel { - displayName: string; - id: string; -} - -const getCredentialAffinityKey = ( - request: NextRequest, - accessKeyId: string | null, -): string | undefined => { - const incoming = getRequestHeaderMap(request.headers); - const conversationId = incoming['x-conversation-id']?.trim(); - - if (!conversationId) { - return undefined; - } - - if (accessKeyId) { - return `access-key:${accessKeyId}:conversation:${conversationId}`; - } - - return `global:conversation:${conversationId}`; -}; - -const toUsageSnapshot = (usage: unknown): UsageSnapshot | null => { - if (!usage || typeof usage !== 'object') { - return null; - } - - return usage as UsageSnapshot; -}; - -const recordProxyUsage = async ({ - model, - proxyContext, - route, - usage, -}: { - model: string; - proxyContext: ProxyContext; - route: string; - usage: unknown; -}): Promise => { - await recordUsageEvent({ - accessKeyId: proxyContext.accessKeyId, - accessKeyName: proxyContext.accessKeyName, - credentialFilename: proxyContext.credentialFilename, - model, - route, - usage: toUsageSnapshot(usage) ?? {}, - }); -}; - -const extractResponsesUsage = (value: unknown): unknown => { - if (!value || typeof value !== 'object') { - return null; - } - - const payload = value as { - response?: { - usage?: unknown; - }; - usage?: unknown; - }; - - return payload.response?.usage ?? payload.usage ?? null; -}; - -const mapResponsesUsageToChat = ( - usage: unknown, -): Record | null => { - if (!usage || typeof usage !== 'object') return null; - - const value = usage as { - cache_creation_input_tokens?: unknown; - cache_read_input_tokens?: unknown; - input_tokens?: unknown; - input_tokens_details?: { - cache_creation_tokens?: unknown; - cached_tokens?: unknown; - }; - output_tokens?: unknown; - output_tokens_details?: { - reasoning_tokens?: unknown; - }; - total_tokens?: unknown; - }; - const inputTokens = Number(value.input_tokens ?? 0); - const outputTokens = Number(value.output_tokens ?? 0); - const cachedTokens = Number( - value.input_tokens_details?.cached_tokens ?? - value.cache_read_input_tokens ?? - 0, - ); - const cacheCreationTokens = Number( - value.input_tokens_details?.cache_creation_tokens ?? - value.cache_creation_input_tokens ?? - 0, - ); - const reasoningTokens = Number( - value.output_tokens_details?.reasoning_tokens ?? 0, - ); - - return { - completion_tokens: outputTokens, - completion_tokens_details: { - reasoning_tokens: reasoningTokens, - }, - prompt_tokens: inputTokens, - prompt_tokens_details: { - cache_creation_tokens: cacheCreationTokens, - cached_tokens: cachedTokens, - }, - total_tokens: Number(value.total_tokens ?? inputTokens + outputTokens), - }; -}; - -const extractResponsesId = (value: unknown): string | null => { - if (!value || typeof value !== 'object') return null; - const payload = value as { - id?: unknown; - response?: { id?: unknown }; - }; - const id = payload.response?.id ?? payload.id; - return typeof id === 'string' && id ? id : null; -}; - -const parseUsageHeader = (response: Response): unknown => { - const usageHeader = response.headers.get('x-codebuddy-usage'); - - if (!usageHeader) { - return null; - } - - try { - return JSON.parse(usageHeader) as unknown; - } catch { - return null; - } -}; - -const trackResponsesUsageStream = async ({ - fallbackUsage, - model, - onResponseId, - proxyContext, - upstreamResponse, -}: { - fallbackUsage: unknown; - model: string; - onResponseId?: (responseId: string) => Promise; - proxyContext: ProxyContext; - upstreamResponse: Response; -}): Promise => { - if (!upstreamResponse.body) { - await recordProxyUsage({ - model, - proxyContext, - route: '/v1/responses', - usage: fallbackUsage, - }); - - return new Response(null, { - headers: upstreamResponse.headers, - status: upstreamResponse.status, - }); - } - - const decoder = new TextDecoder(); - const encoder = new TextEncoder(); - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - const closer = createStreamCloser(); - let latestUsage = fallbackUsage; - let responseBinding: Promise | null = null; - let usageRecorded = false; - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - const recordStreamUsage = async (): Promise => { - if (usageRecorded) return; - usageRecorded = true; - try { - await recordProxyUsage({ - model, - proxyContext, - route: '/v1/responses', - usage: latestUsage, - }); - } catch (error) { - console.error('[CodeBuddy2API] Failed to record Responses stream usage', { - error, - route: '/v1/responses', - }); - } - }; - const bindResponseId = (id: string): Promise => { - if (!onResponseId) return Promise.resolve(); - responseBinding ??= onResponseId(id).catch((error) => { - console.error( - '[CodeBuddy2API] Failed to bind upstream Responses session', - { - error, - responseId: id, - }, - ); - }); - return responseBinding; - }; - const stream = new ReadableStream({ - start: (controller) => { - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - let responseId: string | null = null; - - const inspectFrame = async (frame: string): Promise => { - for (const line of frame.split('\n')) { - if (!line.startsWith('data:')) { - continue; - } - - const raw = line.slice(5).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - try { - const event = JSON.parse(raw) as unknown; - latestUsage = extractResponsesUsage(event) ?? latestUsage; - responseId = extractResponsesId(event) ?? responseId; - if (responseId) await bindResponseId(responseId); - } catch { - // Preserve malformed upstream frames without recording them. - } - } - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - if (buffer) { - await inspectFrame(buffer); - if (closer.closed) return; - controller.enqueue(encoder.encode(buffer)); - } - - await recordStreamUsage(); - await responseBinding; - releaseReader(); - closer.mark(); - controller.close(); - return; - } - - const text = decoder.decode(value, { stream: true }); - buffer += text; - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - buffer = ''; - } - - for (const frame of frames) { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - continue; - } - await inspectFrame(frame); - if (cancelled) return; - controller.enqueue(encoder.encode(`${frame}\n\n`)); - } - } - }; - - void pump().catch(async (error) => { - if (cancelled) return; - console.error('[CodeBuddy2API] Responses upstream stream failed', { - error, - route: '/v1/responses', - }); - await responseBinding; - await recordStreamUsage(); - releaseReader(); - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage !== null) { - // Frames here are forwarded verbatim, so the error has to arrive as - // the Responses protocol's own event. - closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); - return; - } - - controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - await responseBinding; - await recordStreamUsage(); - releaseReader(); - } - }, - }); - - return new Response(stream, { - headers: upstreamResponse.headers, - status: upstreamResponse.status, - }); -}; - -const logUpstreamFailure = ({ - detail, - error, - route, - status, - url, -}: { - detail?: string; - error?: unknown; - route: string; - status?: number; - url: string; -}): void => { - const payload: Record = { - route, - url, - }; - - if (typeof status === 'number') { - payload.status = status; - } - - if (detail) { - payload.detail = detail.slice(0, 1000); - } - - if (error) { - payload.error = error; - } - - console.error('[CodeBuddy2API] Upstream request failed', payload); -}; - -const hasPromptCacheControl = (content: unknown): boolean => { - return ( - Array.isArray(content) && - content.some( - (part) => !!part && typeof part === 'object' && 'cache_control' in part, - ) - ); -}; - -const createCacheableTextBlock = (text: string): CacheableTextBlock => ({ - type: 'text', - text, - cache_control: { type: 'ephemeral' }, -}); - -const addPromptCacheControl = (message: OpenAIMessage): OpenAIMessage => { - if ( - typeof message.content === 'string' && - message.content.trim().length >= MIN_AUTO_CACHE_TEXT_LENGTH - ) { - return { - ...message, - content: [createCacheableTextBlock(message.content)], - }; - } - - if (Array.isArray(message.content)) { - const textIndex = message.content.findIndex( - (part) => - !!part && - typeof part === 'object' && - (part as { type?: unknown }).type === 'text' && - typeof (part as { text?: unknown }).text === 'string' && - (part as { text: string }).text.trim().length >= - MIN_AUTO_CACHE_TEXT_LENGTH, - ); - - if (textIndex >= 0) { - return { - ...message, - content: message.content.map((part, index) => - index === textIndex && part && typeof part === 'object' - ? { - ...part, - cache_control: { type: 'ephemeral' }, - } - : part, - ), - }; - } - } - - return message; -}; - -const applyPromptCacheControl = ( - messages: OpenAIMessage[], -): OpenAIMessage[] => { - const explicitCacheControl = messages.some((message) => - hasPromptCacheControl(message.content), - ); - - if (explicitCacheControl) { - return messages; - } - - const cacheableIndexes = new Set(); - const systemIndex = messages.findIndex( - (message) => message.role === 'system', - ); - - if (systemIndex >= 0) { - cacheableIndexes.add(systemIndex); - } - - let lastUserIndex = -1; - - for (let index = messages.length - 1; index >= 0; index -= 1) { - if (messages[index]?.role === 'user') { - lastUserIndex = index; - break; - } - } - - if (lastUserIndex >= 0) { - cacheableIndexes.add(lastUserIndex); - } - - if (cacheableIndexes.size === 0) { - return messages; - } - - return messages.map((message, index) => - cacheableIndexes.has(index) ? addPromptCacheControl(message) : message, - ); -}; - -const normalizeMessages = ( - messages: OpenAIMessage[], - firstMessageRoleToSystem: boolean, - firstSystemMessageRoleToUser: boolean, -): OpenAIMessage[] => { - const filtered = messages.filter( - (item) => item.role && item.content !== undefined, - ); - - const firstSystemIndex = firstSystemMessageRoleToUser - ? filtered.findIndex((message) => message.role === 'system') - : -1; - const normalized = filtered.map((message, index) => { - if ( - (firstMessageRoleToSystem && message.role === 'developer') || - index === firstSystemIndex - ) { - return { ...message, role: 'user' }; - } - - return message; - }); - - // Preserve role:'tool' messages so the OpenAI-compatible upstream - // receives a valid tool_calls/tool-result pair for multi-step tool loops. - return applyPromptCacheControl(normalized); -}; - -export const resolveProxyContext = async ( - request: NextRequest, - model?: string, -): Promise => { - const accessKey = await resolveRequestAccessKey(request); - const credential = await resolveCredentialForRequest({ - accessKeyId: accessKey?.id, - affinityKey: getCredentialAffinityKey(request, accessKey?.id ?? null), - allowedCredentialFilenames: accessKey?.credentialFilenames, - model, - }); - - if (!credential) { - throw new Error('No valid CodeBuddy credentials found'); - } - - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - if (!bearerToken) { - throw new Error('Saved credential does not include a bearer token'); - } - - return { - accessKeyId: accessKey?.id ?? null, - accessKeyName: accessKey?.name ?? null, - auth: { - type: 'bearer', - bearerToken, - userId: String(credential.data.user_id ?? 'unknown'), - credentialData: credential.data, - }, - credentialFilename: credential.filename, - preferences: getCredentialProxySettings(credential.data), - }; -}; - -export const createProxyContextFromCredential = ( - credential: CredentialRecord, -): ProxyContext => { - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - if (!bearerToken) { - throw new Error('Saved credential does not include a bearer token'); - } - - return { - accessKeyId: null, - accessKeyName: null, - auth: { - type: 'bearer', - bearerToken, - userId: String(credential.data.user_id ?? 'unknown'), - credentialData: credential.data, - }, - credentialFilename: credential.filename, - preferences: getCredentialProxySettings(credential.data), - }; -}; - -export const resolveProxyContextByCredentialFilename = async ( - filename: string, - options?: { - accessKey?: { - id?: string | null; - name?: string | null; - }; - allowedCredentialFilenames?: string[]; - requireEligible?: boolean; - }, -): Promise => { - const credential = options?.requireEligible - ? await findEligibleCredentialRecordByFilename( - filename, - options.allowedCredentialFilenames, - ) - : await findCredentialRecordByFilename(filename); - - if (!credential) { - throw new Error('Selected credential was not found'); - } - - return { - ...createProxyContextFromCredential(credential), - accessKeyId: options?.accessKey?.id ?? null, - accessKeyName: options?.accessKey?.name ?? null, - }; -}; - -const getCredentialValue = ( - value: unknown, - candidateKeys: string[], -): string | number | null => { - if (Array.isArray(value)) { - for (const item of value) { - const nested = getCredentialValue(item, candidateKeys); - - if (nested !== null && nested !== '') { - return nested; - } - } - - return null; - } - - if (value && typeof value === 'object') { - for (const key of candidateKeys) { - const direct = (value as Record)[key]; - - if (direct !== undefined && direct !== null && direct !== '') { - return direct as string | number; - } - } - - for (const nestedValue of Object.values(value as Record)) { - const nested = getCredentialValue(nestedValue, candidateKeys); - - if (nested !== null && nested !== '') { - return nested; - } - } - } - - return null; -}; - -export const buildUpstreamHeaders = async ( - request: NextRequest, - auth: ResolvedAuth, -): Promise => { - const baseUrl = new URL(await getCodeBuddyApiEndpoint()); - const incoming = getRequestHeaderMap(request.headers); - const requestId = - incoming['x-request-id'] ?? crypto.randomUUID().replaceAll('-', ''); - const conversationId = incoming['x-conversation-id'] ?? crypto.randomUUID(); - const conversationRequestId = - incoming['x-conversation-request-id'] ?? - crypto.randomUUID().replaceAll('-', ''); - const conversationMessageId = - incoming['x-conversation-message-id'] ?? - crypto.randomUUID().replaceAll('-', ''); - const headers = new Headers(incoming); - headers.set('Accept', 'application/json'); - headers.set('Authorization', `Bearer ${auth.bearerToken}`); - headers.set('Content-Type', 'application/json'); - headers.set('Host', baseUrl.host); - headers.set('User-Agent', CODEBUDDY_USER_AGENT); - headers.set('X-Agent-Intent', 'craft'); - headers.set('X-Conversation-ID', conversationId); - headers.set('X-Conversation-Message-ID', conversationMessageId); - headers.set('X-Conversation-Request-ID', conversationRequestId); - headers.set('X-IDE-Name', 'CLI'); - headers.set('X-IDE-Type', 'CLI'); - headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); - headers.set('X-Client-Platform', 'web'); - headers.set('X-Product', 'SaaS'); - headers.set('X-Product-Version', CODEBUDDY_CLI_VERSION); - headers.set('X-Request-ID', requestId); - headers.set('X-Requested-With', 'XMLHttpRequest'); - headers.set('X-User-Id', auth.userId); - headers.set('x-stainless-arch', process.arch); - headers.set('x-stainless-lang', 'js'); - headers.set('x-stainless-os', process.platform); - headers.set('x-stainless-package-version', CODEBUDDY_CLI_VERSION); - headers.set('x-stainless-retry-count', '0'); - headers.set('x-stainless-runtime', 'node'); - headers.set('x-stainless-runtime-version', process.version); - - const domain = getCredentialValue(auth.credentialData, ['domain']); - const enterpriseId = getCredentialValue(auth.credentialData, [ - 'enterprise_id', - 'enterpriseId', - ]); - const tenantId = - getCredentialValue(auth.credentialData, ['tenant_id', 'tenantId']) ?? - enterpriseId; - - if (domain) { - headers.set('X-Domain', String(domain)); - } - - if (enterpriseId) { - headers.set('X-Enterprise-Id', String(enterpriseId)); - } - - if (tenantId) { - headers.set('X-Tenant-Id', String(tenantId)); - } - - const origin = String(domain ?? '') - .toLowerCase() - .endsWith('workbuddy.ai') - ? 'https://www.workbuddy.ai' - : 'https://www.codebuddy.cn'; - headers.set('Content-Type', 'application/json'); - headers.set('Origin', origin); - headers.set('Referer', `${origin}/`); - headers.set('User-Agent', CODEBUDDY_USER_AGENT); - headers.set('X-Product', 'SaaS'); - headers.set('X-Requested-With', 'XMLHttpRequest'); - headers.set('X-IDE-Name', 'CLI'); - headers.set('X-IDE-Type', 'CLI'); - headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); - - return headers; -}; - -const headersToRecord = (headers: HeadersInit): Record => { - return Object.fromEntries(new Headers(headers).entries()); -}; - -const buildUpstreamBody = async ( - body: ChatRequestBody, - context: ProxyContext, -): Promise => { - const normalizedMessages = normalizeMessages( - body.messages ?? [], - context.preferences.firstMessageRoleToSystem, - context.preferences.firstSystemMessageRoleToUser, - ); - const maxTokens = body.max_tokens ?? body.max_completion_tokens; - const credentialModels = getCredentialSupportedModels( - context.auth.credentialData, - ); - const model = - typeof body.model === 'string' && body.model.trim() - ? body.model - : (credentialModels[0] ?? (await getDefaultModel())); - - const hyThinking = await resolveHyChatThinking(model, body); - - return { - model, - messages: normalizedMessages, - stream: true, - temperature: body.temperature, - max_tokens: maxTokens, - max_completion_tokens: body.max_completion_tokens ?? maxTokens, - response_format: body.response_format, - top_p: body.top_p, - frequency_penalty: body.frequency_penalty, - presence_penalty: body.presence_penalty, - stop: body.stop, - stream_options: body.stream_options, - tools: body.tools, - tool_choice: body.tool_choice, - parallel_tool_calls: body.parallel_tool_calls, - thinking: hyThinking.thinking, - reasoning_effort: hyThinking.reasoningEffort, - }; -}; - -export const isImageContentPart = (part: unknown): boolean => { - if (!part || typeof part !== 'object') { - return false; - } - - const value = part as { image_url?: unknown; type?: unknown }; - - if (value.type === 'image_url' || value.type === 'input_image') { - return true; - } - - // Accept the shapes an OpenAI-compatible client may send even when `type` - // is absent or unexpected: any part carrying an image URL is an image. - return ( - typeof value.image_url === 'string' || - Boolean( - value.image_url && - typeof value.image_url === 'object' && - typeof (value.image_url as { url?: unknown }).url === 'string', - ) - ); -}; - -/** - * Reads the image URL out of a Responses `input_image` / `image_url` part. - * Returns undefined when the part carries no usable URL, so callers can drop it - * rather than forwarding a block the upstream would reject. - */ -export const extractImageUrl = (part: unknown): string | undefined => { - if (!part || typeof part !== 'object') { - return undefined; - } - - const { image_url: imageUrl } = part as { image_url?: unknown }; - - // `input_image` carries a bare URL string; the OpenAI Chat-style - // `image_url` part nests it under `url`. - if (typeof imageUrl === 'string') { - return imageUrl || undefined; - } - - if ( - imageUrl && - typeof imageUrl === 'object' && - typeof (imageUrl as { url?: unknown }).url === 'string' - ) { - return (imageUrl as { url: string }).url || undefined; - } - - return undefined; -}; - -const stringifyResponsesInputContent = (content: unknown): string => { - if (typeof content === 'string') return content; - if (content === null || content === undefined) return ''; - if (Array.isArray(content)) { - return content - .map((part) => { - if (typeof part === 'string') return part; - if (part && typeof part === 'object' && 'text' in part) { - return String((part as { text?: unknown }).text ?? ''); - } - return JSON.stringify(part); - }) - .join(''); - } - return JSON.stringify(content); -}; - -const mapChatContentToResponses = ( - content: unknown, -): Array> => { - if (!Array.isArray(content)) { - return [ - { - text: stringifyResponsesInputContent(content), - type: 'input_text', - }, - ]; - } - - return content.flatMap((part): Array> => { - if (typeof part === 'string') { - return [{ text: part, type: 'input_text' }]; - } - if (!part || typeof part !== 'object') { - return [{ text: JSON.stringify(part), type: 'input_text' }]; - } - const value = part as { - image_url?: string | { detail?: unknown; url?: unknown }; - text?: unknown; - type?: unknown; - }; - if (value.type === 'image_url') { - const imageUrl = - typeof value.image_url === 'string' - ? value.image_url - : value.image_url?.url; - if (typeof imageUrl === 'string' && imageUrl) { - const detail = - typeof value.image_url === 'object' && - typeof value.image_url.detail === 'string' - ? value.image_url.detail - : undefined; - return [ - { - image_url: imageUrl, - ...(detail ? { detail } : {}), - type: 'input_image', - }, - ]; - } - } - if (value.type === 'input_image' && typeof value.image_url === 'string') { - return [{ image_url: value.image_url, type: 'input_image' }]; - } - if (typeof value.text === 'string') { - return [{ text: value.text, type: 'input_text' }]; - } - return [{ text: JSON.stringify(value), type: 'input_text' }]; - }); -}; - -const translateChatToolChoiceToResponses = (toolChoice: unknown): unknown => { - if (typeof toolChoice === 'string') return toolChoice; - if (!toolChoice || typeof toolChoice !== 'object') return undefined; - const value = toolChoice as { - function?: { name?: unknown }; - name?: unknown; - type?: unknown; - }; - if (value.type !== 'function') return toolChoice; - const name = value.function?.name ?? value.name; - return typeof name === 'string' ? { name, type: 'function' } : toolChoice; -}; - -const translateChatResponseFormatToResponses = ( - responseFormat: unknown, -): Record | undefined => { - if (!responseFormat || typeof responseFormat !== 'object') return undefined; - const value = responseFormat as { - json_schema?: Record; - type?: unknown; - }; - if (value.type === 'json_object') { - return { format: { type: 'json_object' } }; - } - if (value.type !== 'json_schema' || !value.json_schema) return undefined; - const schema = value.json_schema; - if (typeof schema.name !== 'string' || !schema.name) return undefined; - return { - format: { - ...(schema.description ? { description: schema.description } : {}), - name: schema.name, - schema: schema.schema ?? { type: 'object', properties: {} }, - ...(typeof schema.strict === 'boolean' ? { strict: schema.strict } : {}), - type: 'json_schema', - }, - }; -}; - -const translateChatThinkingToResponses = ( - thinking: Record | undefined, - reasoningEffort: string | undefined, -): Record | undefined => { - if (!thinking) - return reasoningEffort ? { effort: reasoningEffort } : undefined; - - if (thinking.type === 'disabled') return { effort: 'none' }; - if (thinking.type !== 'adaptive' && thinking.type !== 'enabled') { - return undefined; - } - - const budgetTokens = - typeof thinking.budget_tokens === 'number' - ? thinking.budget_tokens - : Number.NaN; - const effort = reasoningEffort - ? reasoningEffort - : Number.isFinite(budgetTokens) - ? budgetTokens <= 2_048 - ? 'low' - : budgetTokens <= 8_192 - ? 'medium' - : 'high' - : undefined; - - return { - ...(effort ? { effort } : {}), - summary: 'auto', - }; -}; - -const normalizeStopSequences = ( - stop: string | string[] | undefined, -): string[] => { - return (Array.isArray(stop) ? stop : stop ? [stop] : []).filter(Boolean); -}; - -const findFirstStopSequence = ( - text: string, - stopSequences: string[], -): number | null => { - return stopSequences.reduce((earliest, stopSequence) => { - const index = text.indexOf(stopSequence); - if (index < 0) return earliest; - return earliest === null ? index : Math.min(earliest, index); - }, null); -}; - -const getPendingStopPrefixLength = ( - text: string, - stopSequences: string[], -): number => { - const maximumLength = Math.min( - text.length, - Math.max( - 0, - ...stopSequences.map((stopSequence) => stopSequence.length - 1), - ), - ); - - for (let length = maximumLength; length > 0; length -= 1) { - const suffix = text.slice(-length); - if (stopSequences.some((stopSequence) => stopSequence.startsWith(suffix))) { - return length; - } - } - - return 0; -}; - -/** - * Codex sends `reasoning.effort` in the OpenAI vocabulary, which Hy models do - * not accept, so the effort is rewritten onto the Hy vocabulary before the body - * is forwarded. - */ -const resolveHyResponsesBody = async ( - body: Record, -): Promise> => { - const reasoning = await resolveHyResponsesReasoning( - typeof body.model === 'string' ? body.model : undefined, - body.reasoning as Record | undefined, - ); - - return { ...body, reasoning }; -}; - -const normalizeResponsesUpstreamBody = async ( - body: Record, -): Promise> => { - const { messages, ...rest } = body; - - if (rest.input !== undefined || !Array.isArray(messages)) { - return resolveHyResponsesBody(rest); - } - - const systemInstructions = messages - .filter((message) => { - return ( - message && - typeof message === 'object' && - ((message as { role?: unknown }).role === 'system' || - (message as { role?: unknown }).role === 'developer') - ); - }) - .map((message) => { - return stringifyResponsesInputContent( - (message as { content?: unknown }).content, - ); - }) - .filter(Boolean) - .join('\n\n'); - const input = messages.flatMap((message) => { - if (!message || typeof message !== 'object') return []; - const value = message as { content?: unknown; role?: unknown }; - if (value.role === 'system' || value.role === 'developer') return []; - const role = value.role === 'assistant' ? 'assistant' : 'user'; - return [ - { - content: mapChatContentToResponses(value.content), - role, - }, - ]; - }); - - const existingInstructions = - typeof rest.instructions === 'string' ? rest.instructions.trim() : ''; - const instructions = [existingInstructions, systemInstructions] - .filter(Boolean) - .join('\n\n'); - - return resolveHyResponsesBody({ - ...rest, - ...(instructions ? { instructions } : {}), - input, - }); -}; - -const buildResponsesBodyFromChat = async ( - body: ChatRequestBody, -): Promise> => { - const instructions = body.messages - ?.filter( - (message) => message.role === 'system' || message.role === 'developer', - ) - .map((message) => stringifyResponsesInputContent(message.content)) - .filter(Boolean) - .join('\n\n'); - const input = - body.messages - ?.filter( - (message) => message.role !== 'system' && message.role !== 'developer', - ) - .map((message) => { - if (message.role === 'tool') { - // A tool may return an image, e.g. a screenshot. The upstream - // `function_call_output` carries `output` as structured content, so - // an image part is preserved there; stringifying it would hand the - // model a base64 dump instead of the image. - const toolOutput = Array.isArray(message.content) - ? message.content.filter( - (part) => part !== null && part !== undefined, - ) - : message.content; - const hasImage = Array.isArray(toolOutput) - ? toolOutput.some(isImageContentPart) - : isImageContentPart(toolOutput); - - return { - call_id: message.tool_call_id, - output: hasImage - ? mapChatContentToResponses(toolOutput) - : stringifyResponsesInputContent(toolOutput), - type: 'function_call_output', - }; - } - const toolCalls = Array.isArray(message.tool_calls) - ? message.tool_calls - : []; - const functionCalls = toolCalls.flatMap((toolCall) => { - if (!toolCall || typeof toolCall !== 'object') return []; - const call = toolCall as { - function?: { arguments?: unknown; name?: unknown }; - id?: unknown; - }; - if (typeof call.function?.name !== 'string') return []; - return [ - { - arguments: String(call.function.arguments ?? ''), - call_id: String(call.id ?? crypto.randomUUID()), - name: call.function.name, - type: 'function_call', - }, - ]; - }); - const content = mapChatContentToResponses(message.content); - const hasContent = content.some((part) => { - return ( - (part.type === 'input_text' && Boolean(part.text)) || - (part.type === 'input_image' && Boolean(part.image_url)) - ); - }); - const shouldOmitMessage = - message.role === 'assistant' && - functionCalls.length > 0 && - !hasContent; - - return [ - ...(shouldOmitMessage - ? [] - : [ - { - content, - role: message.role === 'assistant' ? 'assistant' : 'user', - }, - ]), - ...functionCalls, - ]; - }) - .flat() ?? []; - const tools = body.tools?.flatMap((tool) => { - if (!tool || typeof tool !== 'object') return []; - const value = tool as { - function?: Record; - type?: unknown; - }; - const definition: Record = - value.type === 'function' && value.function ? value.function : value; - if (typeof definition.name !== 'string') return []; - return [ - { - ...definition, - parameters: definition.parameters ?? { type: 'object', properties: {} }, - type: 'function', - }, - ]; - }); - const text = translateChatResponseFormatToResponses(body.response_format); - const reasoning = await resolveHyResponsesReasoning( - body.model, - translateChatThinkingToResponses(body.thinking, body.reasoning_effort), - ); - - return { - ...(instructions ? { instructions } : {}), - input, - max_output_tokens: body.max_tokens ?? body.max_completion_tokens, - model: body.model, - parallel_tool_calls: body.parallel_tool_calls, - reasoning, - stream: Boolean(body.stream), - temperature: body.temperature, - top_p: body.top_p, - ...(tools?.length ? { tools } : {}), - ...(body.tool_choice - ? { tool_choice: translateChatToolChoiceToResponses(body.tool_choice) } - : {}), - ...(text ? { text } : {}), - }; -}; - -const getUnsupportedResponsesChatOptions = ( - body: ChatRequestBody, -): string[] => { - return [ - body.frequency_penalty !== undefined ? 'frequency_penalty' : null, - body.presence_penalty !== undefined ? 'presence_penalty' : null, - body.thinking !== undefined && - !translateChatThinkingToResponses(body.thinking, body.reasoning_effort) - ? 'thinking' - : null, - ].filter((name): name is string => Boolean(name)); -}; - -const extractResponsesReasoningText = (output: unknown[]): string => { - return output - .flatMap((item) => { - if (!item || typeof item !== 'object') return []; - const value = item as { - content?: unknown; - summary?: unknown; - type?: unknown; - }; - if (value.type !== 'reasoning') return []; - return [value.summary, value.content].flatMap((parts) => { - if (!Array.isArray(parts)) return []; - return parts.flatMap((part) => { - if (!part || typeof part !== 'object') return []; - const text = (part as { text?: unknown }).text; - return typeof text === 'string' ? [text] : []; - }); - }); - }) - .join(''); -}; - -const mapResponsesPayloadToChat = ( - payload: Record, - model: string, - stop: string | string[] | undefined, -): Record => { - const output = Array.isArray(payload.output) ? payload.output : []; - const toolCalls = output.flatMap((item) => { - if (!item || typeof item !== 'object') return []; - const value = item as Record; - if ( - value.type !== 'function_call' && - value.type !== 'mcp_call' && - value.type !== 'custom_tool_call' - ) { - return []; - } - const isCustomToolCall = value.type === 'custom_tool_call'; - const customArguments = JSON.stringify({ - input: String(value.input ?? value.arguments ?? ''), - }); - return [ - { - function: { - arguments: String( - isCustomToolCall ? customArguments : (value.arguments ?? ''), - ), - name: String(value.name ?? 'function'), - }, - id: String(value.call_id ?? value.id ?? crypto.randomUUID()), - type: 'function', - }, - ]; - }); - const usage = - payload.usage && typeof payload.usage === 'object' - ? (payload.usage as Record) - : undefined; - const inputTokens = Number(usage?.input_tokens ?? 0); - const outputTokens = Number(usage?.output_tokens ?? 0); - - const rawOutputText = - typeof payload.output_text === 'string' - ? payload.output_text - : output - .flatMap((item) => { - if (!item || typeof item !== 'object') return []; - const content = (item as { content?: unknown }).content; - if (!Array.isArray(content)) return []; - return content.flatMap((part) => { - if (!part || typeof part !== 'object') return []; - const value = part as { text?: unknown; type?: unknown }; - return value.type === 'output_text' && - typeof value.text === 'string' - ? [value.text] - : []; - }); - }) - .join(''); - const stopIndex = findFirstStopSequence( - rawOutputText, - normalizeStopSequences(stop), - ); - const outputText = - stopIndex === null ? rawOutputText : rawOutputText.slice(0, stopIndex); - const reasoningText = extractResponsesReasoningText(output); - const incompleteReason = - payload.incomplete_details && typeof payload.incomplete_details === 'object' - ? (payload.incomplete_details as { reason?: unknown }).reason - : undefined; - const finishReason = - payload.status === 'incomplete' - ? incompleteReason === 'content_filter' - ? 'content_filter' - : 'length' - : toolCalls.length - ? 'tool_calls' - : 'stop'; - - return { - choices: [ - { - finish_reason: finishReason, - index: 0, - message: { - content: outputText || null, - role: 'assistant', - ...(reasoningText ? { reasoning_content: reasoningText } : {}), - ...(toolCalls.length ? { tool_calls: toolCalls } : {}), - }, - }, - ], - created: Number(payload.created_at ?? Math.floor(Date.now() / 1000)), - id: String(payload.id ?? `chatcmpl-${crypto.randomUUID()}`), - model, - object: 'chat.completion', - usage: { - completion_tokens: outputTokens, - prompt_tokens: inputTokens, - total_tokens: Number(usage?.total_tokens ?? inputTokens + outputTokens), - }, - }; -}; - -const mapResponsesStreamToChat = ( - upstreamResponse: Response, - model: string, - proxyContext: ProxyContext, - route: string, - stop: string | string[] | undefined, - includeUsage: boolean, -): Response => { - const closer = createStreamCloser(); - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const responseId = `chatcmpl-${crypto.randomUUID()}`; - let reader: ReadableStreamDefaultReader | null = - upstreamResponse.body?.getReader() ?? null; - const fallbackUsage = parseUsageHeader(upstreamResponse); - let buffer = ''; - let emittedFinish = false; - let emittedUsage = false; - let hasToolCalls = false; - let latestUsage = fallbackUsage; - let usageRecorded = false; - const stopSequences = normalizeStopSequences(stop); - let pendingStopText = ''; - const toolIndexes = new Map(); - const toolCallIds = new Map(); - const customToolCallIds = new Set(); - const closedCustomToolCallIds = new Set(); - let nextToolIndex = 0; - let stoppedLocally = false; - - const getToolIndex = (itemId: string): number => { - const existing = toolIndexes.get(itemId); - if (existing !== undefined) return existing; - const index = nextToolIndex; - nextToolIndex += 1; - toolIndexes.set(itemId, index); - return index; - }; - - const encodeChunk = (choice: Record): Uint8Array => { - return encoder.encode( - `data: ${JSON.stringify({ - choices: [choice], - created: Math.floor(Date.now() / 1000), - id: responseId, - model, - object: 'chat.completion.chunk', - })}\n\n`, - ); - }; - - const enqueueUsage = ( - controller: ReadableStreamDefaultController, - ): void => { - if (!includeUsage || emittedUsage) return; - const usage = mapResponsesUsageToChat(latestUsage); - if (!usage) return; - - emittedUsage = true; - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ - choices: [], - created: Math.floor(Date.now() / 1000), - id: responseId, - model, - object: 'chat.completion.chunk', - usage, - })}\n\n`, - ), - ); - }; - - const recordStreamUsage = async (): Promise => { - if (usageRecorded) return; - usageRecorded = true; - try { - await recordProxyUsage({ - model, - proxyContext, - route, - usage: latestUsage, - }); - } catch (error) { - console.error('[CodeBuddy2API] Failed to record Responses stream usage', { - error, - route, - }); - } - }; - - const cancelAndReleaseReader = async (reason?: unknown): Promise => { - try { - await reader?.cancel(reason); - } catch (error) { - console.error('[CodeBuddy2API] Failed to cancel Responses stream', { - error, - route, - }); - } finally { - reader?.releaseLock(); - reader = null; - } - }; - - const emitCustomToolCallClosures = ( - controller: ReadableStreamDefaultController, - ): void => { - customToolCallIds.forEach((itemId) => { - if (closedCustomToolCallIds.has(itemId)) return; - const index = getToolIndex(itemId); - const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; - controller.enqueue( - encodeChunk({ - delta: { - tool_calls: [{ function: { arguments: '"}' }, id: callId, index }], - }, - index: 0, - }), - ); - closedCustomToolCallIds.add(itemId); - }); - }; - - const stream = new ReadableStream({ - async pull(controller) { - if (!reader) { - await recordStreamUsage(); - controller.close(); - return; - } - while (true) { - let readResult: ReadableStreamReadResult; - try { - readResult = await reader.read(); - } catch (error) { - await recordStreamUsage(); - reader.releaseLock(); - reader = null; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage !== null) { - closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); - return; - } - - controller.error(error); - return; - } - const { done, value } = readResult; - if (done) { - if (pendingStopText) { - controller.enqueue( - encodeChunk({ - delta: { content: pendingStopText }, - index: 0, - }), - ); - pendingStopText = ''; - } - emitCustomToolCallClosures(controller); - if (!emittedFinish) { - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: hasToolCalls ? 'tool_calls' : 'stop', - index: 0, - }), - ); - } - enqueueUsage(controller); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - await recordStreamUsage(); - reader.releaseLock(); - reader = null; - controller.close(); - return; - } - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split(/\r?\n\r?\n/); - buffer = frames.pop() ?? ''; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - await cancelAndReleaseReader(); - await recordStreamUsage(); - controller.close(); - return; - } - let emitted = false; - for (const frame of frames) { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - await cancelAndReleaseReader(); - await recordStreamUsage(); - controller.close(); - return; - } - const dataLine = frame - .split(/\r?\n/) - .find((line) => line.startsWith('data: ')); - if (!dataLine || dataLine === 'data: [DONE]') continue; - // The upstream here is the chat pipeline, which reports a deadline as - // a terminal error chunk and closes cleanly. Without this the failure - // would be reported to the client as an empty successful response. - const upstreamError = readTimeoutFrame(frame); - - if (upstreamError !== null) { - closer.fail(controller, chatStreamErrorChunks(upstreamError)); - await cancelAndReleaseReader(); - await recordStreamUsage(); - return; - } - try { - const event = JSON.parse(dataLine.slice(6)) as { - delta?: unknown; - item?: unknown; - item_id?: unknown; - output_index?: unknown; - error?: unknown; - response?: unknown; - type?: unknown; - }; - latestUsage = extractResponsesUsage(event) ?? latestUsage; - if ( - stoppedLocally && - event.type !== 'response.completed' && - event.type !== 'response.incomplete' - ) { - continue; - } - if (event.type === 'response.output_text.delta') { - const delta = String(event.delta ?? ''); - if (stopSequences.length) { - pendingStopText += delta; - const stopIndex = findFirstStopSequence( - pendingStopText, - stopSequences, - ); - if (stopIndex !== null) { - const content = pendingStopText.slice(0, stopIndex); - if (content) { - controller.enqueue( - encodeChunk({ delta: { content }, index: 0 }), - ); - } - pendingStopText = ''; - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: 'stop', - index: 0, - }), - ); - emittedFinish = true; - stoppedLocally = true; - emitted = true; - continue; - } - - const pendingLength = getPendingStopPrefixLength( - pendingStopText, - stopSequences, - ); - const content = pendingStopText.slice( - 0, - pendingStopText.length - pendingLength, - ); - pendingStopText = pendingLength - ? pendingStopText.slice(-pendingLength) - : ''; - if (!content) continue; - controller.enqueue( - encodeChunk({ delta: { content }, index: 0 }), - ); - emitted = true; - continue; - } - controller.enqueue( - encodeChunk({ - delta: { content: delta }, - index: 0, - }), - ); - emitted = true; - continue; - } - if ( - event.type === 'response.reasoning_summary_text.delta' || - event.type === 'response.reasoning_text.delta' - ) { - controller.enqueue( - encodeChunk({ - delta: { reasoning_content: String(event.delta ?? '') }, - index: 0, - }), - ); - emitted = true; - continue; - } - if ( - event.type === 'response.output_item.added' && - event.item && - typeof event.item === 'object' - ) { - const item = event.item as { - arguments?: unknown; - call_id?: unknown; - id?: unknown; - input?: unknown; - name?: unknown; - type?: unknown; - }; - if ( - item.type !== 'function_call' && - item.type !== 'mcp_call' && - item.type !== 'custom_tool_call' - ) { - continue; - } - const isCustomToolCall = item.type === 'custom_tool_call'; - const initialArguments = isCustomToolCall - ? `{"input":"${JSON.stringify( - String(item.input ?? item.arguments ?? ''), - ).slice(1, -1)}` - : String(item.arguments ?? ''); - const itemId = String(item.id ?? item.call_id ?? nextToolIndex); - const index = getToolIndex(itemId); - const callId = String(item.call_id ?? item.id ?? itemId); - toolCallIds.set(itemId, callId); - if (isCustomToolCall) { - customToolCallIds.add(itemId); - } - hasToolCalls = true; - controller.enqueue( - encodeChunk({ - delta: { - tool_calls: [ - { - function: { - arguments: initialArguments, - name: String(item.name ?? 'function'), - }, - id: callId, - index, - type: 'function', - }, - ], - }, - index: 0, - }), - ); - emitted = true; - continue; - } - if ( - event.type === 'response.function_call_arguments.delta' || - event.type === 'response.mcp_call_arguments.delta' || - event.type === 'response.custom_tool_call_input.delta' - ) { - const itemId = String( - event.item_id ?? event.output_index ?? nextToolIndex, - ); - const index = getToolIndex(itemId); - const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; - toolCallIds.set(itemId, callId); - hasToolCalls = true; - const argumentDelta = - event.type === 'response.custom_tool_call_input.delta' - ? JSON.stringify(String(event.delta ?? '')).slice(1, -1) - : String(event.delta ?? ''); - controller.enqueue( - encodeChunk({ - delta: { - tool_calls: [ - { - function: { arguments: argumentDelta }, - id: callId, - index, - }, - ], - }, - index: 0, - }), - ); - emitted = true; - continue; - } - if (event.type === 'response.completed') { - if (pendingStopText) { - controller.enqueue( - encodeChunk({ - delta: { content: pendingStopText }, - index: 0, - }), - ); - pendingStopText = ''; - } - emitCustomToolCallClosures(controller); - if (!emittedFinish) { - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: hasToolCalls ? 'tool_calls' : 'stop', - index: 0, - }), - ); - } - enqueueUsage(controller); - emittedFinish = true; - emitted = true; - if (stoppedLocally) { - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - await cancelAndReleaseReader('Stop sequence matched'); - await recordStreamUsage(); - controller.close(); - return; - } - continue; - } - if (event.type === 'response.incomplete') { - if (pendingStopText) { - controller.enqueue( - encodeChunk({ - delta: { content: pendingStopText }, - index: 0, - }), - ); - pendingStopText = ''; - } - emitCustomToolCallClosures(controller); - const incompleteReason = - event.response && typeof event.response === 'object' - ? ( - (event.response as { incomplete_details?: unknown }) - .incomplete_details as { reason?: unknown } | undefined - )?.reason - : undefined; - if (!emittedFinish) { - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: - incompleteReason === 'content_filter' - ? 'content_filter' - : 'length', - index: 0, - }), - ); - } - enqueueUsage(controller); - emittedFinish = true; - emitted = true; - if (stoppedLocally) { - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - await cancelAndReleaseReader('Stop sequence matched'); - await recordStreamUsage(); - controller.close(); - return; - } - continue; - } - if ( - event.type === 'response.failed' || - event.type === 'response.error' || - event.type === 'error' - ) { - const failure = - event.error ?? - (event.response && typeof event.response === 'object' - ? (event.response as { error?: unknown }).error - : undefined); - const message = - failure && typeof failure === 'object' - ? String( - (failure as { message?: unknown }).message ?? failure, - ) - : String(failure ?? 'Upstream Responses stream failed'); - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ error: { message } })}\n\n`, - ), - ); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - await cancelAndReleaseReader(); - await recordStreamUsage(); - controller.close(); - return; - } - } catch { - // Ignore malformed upstream events and continue reading. - } - } - if (stoppedLocally) continue; - if (emitted) return; - } - }, - async cancel(reason) { - await cancelAndReleaseReader(reason); - await recordStreamUsage(); - }, - }); - - return new Response(stream, { - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - status: upstreamResponse.status, - }); -}; - -const aggregateToolCalls = ( - toolCalls: NonNullable, -): Array<{ - id?: string; - type?: string; - function: { - arguments: string; - name: string; - }; -}> => { - const aggregated = new Map< - string, - { - order: number; - id?: string; - type?: string; - function: { - arguments: string; - name: string; - }; - } - >(); - const latestKeyByIndex = new Map(); - - toolCalls.forEach((toolCall, position) => { - const normalizedId = createNormalizedToolCallId(toolCall.id, position); - const key = - (toolCall.id ? `id:${normalizedId}` : undefined) ?? - (typeof toolCall.index === 'number' - ? latestKeyByIndex.get(toolCall.index) - : undefined) ?? - `position:${position}`; - const current = aggregated.get(key) ?? { - order: aggregated.size, - function: { - arguments: '', - name: '', - }, - }; - - if (toolCall.id) { - current.id = normalizedId; - } - - if (toolCall.type) { - current.type = toolCall.type; - } - - if (toolCall.function?.name) { - current.function.name += toolCall.function.name; - } - - if (toolCall.function?.arguments) { - current.function.arguments += toolCall.function.arguments; - } - - aggregated.set(key, current); - - if (typeof toolCall.index === 'number') { - latestKeyByIndex.set(toolCall.index, key); - } - }); - - return [...aggregated.values()] - .sort((left, right) => left.order - right.order) - .map(({ order: _order, ...value }, index) => ({ - ...value, - id: value.id ?? createNormalizedToolCallId(undefined, index), - })); -}; - -const getToolCallStateKey = ( - toolCall: ToolCallChunk, - position: number, -): string => { - if (toolCall.id) { - return `id:${toolCall.id}`; - } - - if (typeof toolCall.index === 'number') { - return `index:${toolCall.index}`; - } - - return `position:${position}`; -}; - -const createNormalizedToolCallId = ( - sourceId: string | undefined, - normalizedIndex: number, -): string => { - if (sourceId && !sourceId.startsWith('tooluse_')) { - return sourceId; - } - - const suffix = - sourceId?.replace(/^tooluse_/, '') ?? - `${normalizedIndex}_${crypto.randomUUID().replaceAll('-', '')}`; - - return `call_${suffix}`; -}; - -const resolveToolCallMapping = ( - state: ToolCallNormalizationState, - toolCall: ToolCallChunk, - position: number, -): ToolCallMapping => { - const keys = toolCall.id - ? [`id:${toolCall.id}`] - : [ - typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, - `position:${position}`, - ].filter((value): value is string => value !== null); - const existing = keys - .map((key) => state.mappings.get(key)) - .find((value) => value !== undefined); - - if (existing) { - return existing; - } - - return { - id: createNormalizedToolCallId(toolCall.id, state.nextIndex), - index: state.nextIndex++, - }; -}; - -const normalizeStreamToolCalls = ( - chunk: ChatStreamChunk, - state: ToolCallNormalizationState, -): ChatStreamChunk => { - if (!chunk.choices?.length) { - return chunk; - } - - return { - ...chunk, - choices: chunk.choices.map((choice) => { - if (!choice.delta?.tool_calls?.length) { - return choice; - } - - return { - ...choice, - delta: { - ...choice.delta, - tool_calls: choice.delta.tool_calls.map((toolCall, position) => { - const mapping = resolveToolCallMapping(state, toolCall, position); - const sourceKey = getToolCallStateKey(toolCall, position); - - state.mappings.set(sourceKey, mapping); - - if (toolCall.id) { - state.mappings.set(`id:${toolCall.id}`, mapping); - } - - if (typeof toolCall.index === 'number') { - state.mappings.set(`index:${toolCall.index}`, mapping); - } - - return { - ...toolCall, - id: mapping.id, - index: mapping.index, - }; - }), - }, - }; - }), - }; -}; - -const normalizeStreamingResponse = ({ - model, - proxyContext, - route, - upstreamResponse, -}: { - model: string; - proxyContext: ProxyContext; - route: string; - upstreamResponse: Response; -}): Response => { - if (!upstreamResponse.body) { - return new Response(null, { - status: upstreamResponse.status, - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); - } - - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const state: ToolCallNormalizationState = { - mappings: new Map(), - nextIndex: 0, - }; - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - const closer = createStreamCloser(); - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - - const stream = new ReadableStream({ - start: (controller) => { - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - let latestUsage: unknown = null; - - const processFrame = (frame: string): string => { - const lines = frame.split('\n'); - const lineIndex = lines.findIndex((line) => line.startsWith('data: ')); - - if (lineIndex === -1) { - return frame; - } - - const raw = lines[lineIndex]?.slice(6).trim() ?? ''; - - if (!raw || raw === '[DONE]') { - return frame; - } - - try { - const chunk = JSON.parse(raw) as ChatStreamChunk; - if (chunk.usage !== undefined) { - latestUsage = chunk.usage; - } - const normalized = normalizeStreamToolCalls(chunk, state); - lines[lineIndex] = `data: ${JSON.stringify(normalized)}`; - return lines.join('\n'); - } catch { - return frame; - } - }; - - const flushFrames = (frames: string[]): void => { - frames.forEach((frame) => { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - return; - } - controller.enqueue(encoder.encode(`${processFrame(frame)}\n\n`)); - }); - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - if (buffer.trim()) { - flushFrames([buffer]); - } - - await recordProxyUsage({ - model, - proxyContext, - route, - usage: latestUsage, - }); - - releaseReader(); - try { - controller.close(); - } catch { - // The downstream stream may have been cancelled while the pump was completing. - } - return; - } - - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - try { - await reader!.cancel(); - } finally { - releaseReader(); - controller.close(); - } - return; - } - flushFrames(frames); - } - }; - - void pump().catch((error) => { - if (cancelled) return; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage !== null) { - closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); - return; - } - - controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - releaseReader(); - } - }, - }); - - return new Response(stream, { - status: upstreamResponse.status, - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -}; - -const aggregateUpstreamStream = async ( - upstreamResponse: Response, - fallbackModel: string, -): Promise<{ model: string; response: Response; usage: unknown }> => { - const payloadText = await upstreamResponse.text(); - const toolCalls: NonNullable = []; - let responseId = ''; - let responseObject = 'chat.completion'; - let created = Math.floor(Date.now() / 1000); - let model = fallbackModel; - let content = ''; - let reasoningContent = ''; - let finishReason: string | null = 'stop'; - let role = 'assistant'; - let usage: unknown = null; - - for (const frame of payloadText.split('\n\n')) { - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - continue; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - let chunk: ChatStreamChunk; - - try { - chunk = JSON.parse(raw) as ChatStreamChunk; - } catch { - return { - model: fallbackModel, - response: createErrorResponse( - 502, - 'Failed to parse upstream SSE frame', - ), - usage: null, - }; - } - - if (chunk.id) { - responseId = chunk.id; - } - - if (chunk.object) { - responseObject = chunk.object.replace(/\.chunk$/, ''); - } - - if (typeof chunk.created === 'number') { - created = chunk.created; - } - - if (chunk.model) { - model = chunk.model; - } - - if (chunk.usage !== undefined) { - usage = chunk.usage; - } - - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - - if (delta?.role) { - role = delta.role; - } - - if (delta?.content) { - content += delta.content; - } - - if (delta?.reasoning_content ?? delta?.reasoning) { - reasoningContent += delta.reasoning_content ?? delta.reasoning; - } - - if (delta?.tool_calls?.length) { - toolCalls.push(...delta.tool_calls); - } - - if (choice?.finish_reason !== undefined) { - finishReason = choice.finish_reason ?? finishReason; - } - } - - const aggregatedToolCalls = aggregateToolCalls(toolCalls); - const message: Record = { - role, - content: content || null, - }; - - if (reasoningContent) { - message.reasoning_content = reasoningContent; - } - - if (aggregatedToolCalls.length) { - message.tool_calls = aggregatedToolCalls; - } - - return { - model, - response: Response.json({ - id: responseId || `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`, - object: responseObject, - created, - model, - choices: [ - { - index: 0, - message, - finish_reason: - finishReason ?? - (aggregatedToolCalls.length ? 'tool_calls' : 'stop'), - }, - ], - usage, - }), - usage, - }; -}; - -const isServerWebToolName = (name: string): boolean => { - const normalized = normalizeToolName(name); - - return ( - normalized === normalizeToolName(WEB_SEARCH_TOOL_NAME) || - normalized === normalizeToolName(WEB_FETCH_TOOL_NAME) - ); -}; - -const SERVER_WEB_TOOL_NAMES = [ - normalizeToolName(WEB_SEARCH_TOOL_NAME), - normalizeToolName(WEB_FETCH_TOOL_NAME), -]; - -interface StreamProbeState { - toolNames: Map; -} - -const mergeToolName = (previous: string, incoming: string): string => { - if (!previous || incoming.startsWith(previous)) { - return incoming; - } - - if (!incoming || previous.endsWith(incoming)) { - return previous; - } - - return previous + incoming; -}; - -const classifyStreamFrame = ( - frame: string, - state: StreamProbeState, - serverToolNames: string[], -): 'passthrough' | 'server-tool' | null => { - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - return null; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - return raw === '[DONE]' ? 'passthrough' : null; - } - - try { - const chunk = JSON.parse(raw) as ChatStreamChunk; - - for (const choice of chunk.choices ?? []) { - const delta = choice.delta; - const toolCalls = delta?.tool_calls ?? []; - let hasNonServerTool = false; - - for (const [position, toolCall] of toolCalls.entries()) { - const incoming = toolCall.function?.name; - - if (typeof incoming !== 'string' || !incoming) { - continue; - } - - const key = - typeof toolCall.index === 'number' - ? `index:${toolCall.index}` - : toolCall.id - ? `id:${toolCall.id}` - : `position:${position}`; - const name = mergeToolName(state.toolNames.get(key) ?? '', incoming); - const normalized = normalizeToolName(name); - - state.toolNames.set(key, name); - - if (isServerWebToolName(name) && serverToolNames.includes(normalized)) { - return 'server-tool'; - } - - if ( - normalized && - !serverToolNames.some((serverName) => - serverName.startsWith(normalized), - ) - ) { - hasNonServerTool = true; - } - } - - if ( - delta?.content || - delta?.reasoning_content || - delta?.reasoning || - hasNonServerTool || - choice.finish_reason != null - ) { - return 'passthrough'; - } - } - } catch { - return null; - } - - return null; -}; - -const concatenateChunks = (chunks: Uint8Array[]): ArrayBuffer => { - const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); - const combined = new Uint8Array(size); - let offset = 0; - - for (const chunk of chunks) { - combined.set(chunk, offset); - offset += chunk.byteLength; - } - - return combined.buffer; -}; - -const createResponseWithBody = (body: BodyInit, response: Response): Response => - new Response(body, { - headers: response.headers, - status: response.status, - statusText: response.statusText, - }); - -const createReplayStreamResponse = ({ - chunks, - reader, - response, -}: { - chunks: Uint8Array[]; - reader: ReadableStreamDefaultReader; - response: Response; -}): Response => { - let cancelled = false; - - const stream = new ReadableStream({ - start: (controller) => { - for (const chunk of chunks) { - controller.enqueue(chunk); - } - - const pump = async (): Promise => { - while (true) { - const { done, value } = await reader.read(); - - if (cancelled) { - return; - } - - if (done) { - reader.releaseLock(); - controller.close(); - return; - } - - controller.enqueue(value); - } - }; - - void pump().catch((error) => { - if (!cancelled) { - controller.error(error); - } - }); - }, - async cancel(reason): Promise { - cancelled = true; - try { - await reader.cancel(reason); - } finally { - reader.releaseLock(); - } - }, - }); - - return createResponseWithBody(stream, response); -}; - -const detectServerToolStream = async ( - response: Response, - fallbackModel: string, - serverToolNames: string[], -): Promise => { - if (!response.body) { - return response; - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - const chunks: Uint8Array[] = []; - const state: StreamProbeState = { toolNames: new Map() }; - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - - if (done) { - reader.releaseLock(); - return createResponseWithBody(concatenateChunks(chunks), response); - } - - chunks.push(value); - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop() ?? ''; - - for (const frame of frames) { - const classification = classifyStreamFrame(frame, state, serverToolNames); - - if (classification === 'passthrough') { - return createReplayStreamResponse({ chunks, reader, response }); - } - - if (classification === 'server-tool') { - while (true) { - const remainder = await reader.read(); - - if (remainder.done) { - reader.releaseLock(); - break; - } - - chunks.push(remainder.value); - } - - return ( - await aggregateUpstreamStream( - new Response(concatenateChunks(chunks)), - fallbackModel, - ) - ).response; - } - } - } -}; - -export const getModelsForCredential = async ({ - bearerToken, - credentialData, -}: { - bearerToken: string; - credentialData: CredentialData; -}): Promise => { - const configuredEndpoint = await getCodeBuddyApiEndpoint(); - const headers = new Headers({ - Accept: 'application/json', - Authorization: `Bearer ${bearerToken}`, - 'X-Product': 'SaaS', - }); - const domain = getCredentialValue(credentialData, ['domain']); - const apiEndpoint = String(domain ?? '') - .toLowerCase() - .endsWith('workbuddy.ai') - ? 'https://www.workbuddy.ai' - : configuredEndpoint; - const enterpriseId = getCredentialValue(credentialData, [ - 'enterprise_id', - 'enterpriseId', - ]); - const tenantId = - getCredentialValue(credentialData, ['tenant_id', 'tenantId']) ?? - enterpriseId; - const userId = getCredentialValue(credentialData, ['user_id', 'userId']); - - if (domain) { - headers.set('X-Domain', String(domain)); - } - - if (enterpriseId) { - headers.set('X-Enterprise-Id', String(enterpriseId)); - } - - if (tenantId) { - headers.set('X-Tenant-Id', String(tenantId)); - } - if (userId) { - headers.set('X-User-Id', String(userId)); - } - - const fetchModels = async (path: string): Promise => - fetch(new URL(path, apiEndpoint), { - headers, - signal: AbortSignal.timeout(15_000), - }); - let response = await fetchModels('/v3/config'); - - if ([400, 404, 405].includes(response.status)) { - // Upstream splits this route by account scope: enterprise accounts must hit - // their own segment, otherwise they are served the personal model catalog. - const enterpriseScope = String(enterpriseId ?? '').trim() || 'personal'; - response = await fetchModels( - `/console/enterprises/${encodeURIComponent(enterpriseScope)}/models`, - ); - } - - if (!response.ok) { - throw new Error(`Model discovery failed with status ${response.status}`); - } - - const payload = (await response.json()) as { - code?: unknown; - data?: { - agents?: Array<{ models?: unknown; name?: unknown }>; - models?: Array<{ disabled?: unknown; id?: unknown; name?: unknown }>; - }; - }; - - if (payload.code !== 0) { - throw new Error('Model discovery returned an unsuccessful response'); - } - - const cliModels = payload.data?.agents?.find( - (agent) => agent.name === 'cli', - )?.models; - const modelsById = new Map( - (payload.data?.models ?? []).flatMap((model) => { - const id = typeof model.id === 'string' ? model.id.trim() : ''; - - if (!id || model.disabled === true) { - return []; - } - - return [ - [ - id, - { - displayName: - typeof model.name === 'string' && model.name.trim() - ? model.name - : id, - id, - }, - ] as const, - ]; - }), - ); - const declaredModelIds = new Set( - (payload.data?.models ?? []) - .map((model) => (typeof model.id === 'string' ? model.id.trim() : '')) - .filter(Boolean), - ); - - if (!Array.isArray(cliModels)) { - return []; - } - - return cliModels.flatMap((modelId) => { - if (typeof modelId !== 'string') { - return []; - } - - const model = modelsById.get(modelId); - if (!model && declaredModelIds.has(modelId)) { - return []; - } - return [ - model ?? { - displayName: modelId, - id: modelId, - }, - ]; - }); -}; - -export const getModelsForCredentials = async ( - credentials: CredentialRecord[], -): Promise => { - const settled = await Promise.allSettled( - credentials.map((credential) => { - const supportedModels = getCredentialSupportedModels(credential.data); - - if (supportedModels.length) { - return Promise.resolve( - supportedModels.map((id) => ({ displayName: id, id })), - ); - } - - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - return bearerToken - ? getModelsForCredential({ - bearerToken, - credentialData: credential.data, - }) - : Promise.resolve([]); - }), - ); - const models = new Map(); - - settled.forEach((result) => { - if (result.status !== 'fulfilled') { - return; - } - - result.value.forEach((model) => { - models.set(model.id, model); - }); - }); - - return [...models.values()].sort((left, right) => - left.id.localeCompare(right.id), - ); -}; - -export const getModelsByCredential = async ( - credentials: CredentialRecord[], -): Promise< - Record -> => { - const results = await Promise.all( - credentials.map(async (credential) => { - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - try { - const models = bearerToken - ? await getModelsForCredential({ - bearerToken, - credentialData: credential.data, - }) - : []; - - return [credential.filename, { error: null, models }] as const; - } catch (error) { - return [ - credential.filename, - { - error: - error instanceof Error ? error.message : 'Model discovery failed', - models: [], - }, - ] as const; - } - }), - ); - - return Object.fromEntries(results); -}; - -export const getModelsResponse = async ( - request?: NextRequest, -): Promise => { - const accessKey = request ? await resolveRequestAccessKey(request) : null; - const models = ( - await getModelsForCredentials( - await listEligibleCredentialRecords(accessKey?.credentialFilenames), - ) - ).map((model) => ({ - id: model.id, - slug: model.id, - display_name: model.displayName, - object: 'model', - created: 0, - owned_by: 'codebuddy', - })); - - return Response.json({ - object: 'list', - data: models, - models, - }); -}; + aggregateUpstreamStream, + normalizeStreamingResponse, +} from './codebuddy/chat-stream'; +import { resolveProxyContext } from './codebuddy/context'; +import { + buildResponsesBodyFromChat, + getUnsupportedResponsesChatOptions, + normalizeResponsesUpstreamBody, +} from './codebuddy/responses-request'; +import { + mapResponsesPayloadToChat, + mapResponsesStreamToChat, +} from './codebuddy/responses-response'; +import { detectServerToolStream } from './codebuddy/server-tools'; +import { + SERVER_WEB_TOOL_NAMES, + type ChatRequestBody, + type ProxyContext, +} from './codebuddy/types'; +import { + extractResponsesId, + extractResponsesUsage, + logUpstreamFailure, + parseUsageHeader, + recordProxyUsage, +} from './codebuddy/usage'; +import { trackResponsesUsageStream } from './codebuddy/usage-stream'; +import { + buildUpstreamBody, + buildUpstreamHeaders, + headersToRecord, +} from './codebuddy/upstream'; +import { + enqueueUpstreamResponseSnapshot, + setDebugTraceCredential, + setDebugTraceError, + setDebugUpstreamRequest, + type DebugTrace, +} from '../domain/debug'; +import { + getApiFirstDeltaTimeoutMs, + getCodeBuddyApiEndpoint, + getDefaultModel, +} from '../domain/config'; +import { withCodeBuddyToken } from '../search/token'; +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../search/tool'; +import { createErrorResponse } from '../shared/http'; +import { fetchWithDeadline } from '../shared/upstream-timeout'; +import { + attachServerToolExecutions, + attachServerToolTurns, + type ChatCompletionPayload, + executeWebSearchLoop, + type ServerToolCallbacks, + synthesizeChatCompletionStream, +} from './web-search-loop'; /** * One round trip to `/v2/chat/completions`. Split out from @@ -3515,3 +664,27 @@ export const proxyResponsesUpstream = async ( ); } }; + +// Re-exported for the importers that reached these through this module before +// the split: the route handlers, the domain layer, and the test suite. +export type { + ChatRequestBody, + DiscoveredModel, + ProxyContext, +} from './codebuddy/types'; +export { + createProxyContextFromCredential, + resolveProxyContext, + resolveProxyContextByCredentialFilename, +} from './codebuddy/context'; +export { buildUpstreamHeaders } from './codebuddy/upstream'; +export { + extractImageUrl, + isImageContentPart, +} from './codebuddy/responses-request'; +export { + getModelsByCredential, + getModelsForCredential, + getModelsForCredentials, + getModelsResponse, +} from './codebuddy/models'; diff --git a/lib/server/proxy/codebuddy/chat-stream.ts b/lib/server/proxy/codebuddy/chat-stream.ts new file mode 100644 index 0000000..07c403e --- /dev/null +++ b/lib/server/proxy/codebuddy/chat-stream.ts @@ -0,0 +1,474 @@ +import { createErrorResponse } from '../../shared/http'; +import { createSseResponse } from '../../shared/sse'; +import { + chatStreamErrorChunks, + createStreamCloser, + toUpstreamTimeoutMessage, +} from '../../shared/upstream-timeout'; +import { recordProxyUsage } from './usage'; +import { + type ChatStreamChunk, + type ChatStreamDelta, + CORS_HEADERS, + MAX_STREAM_FRAME_LENGTH, + type ProxyContext, + type ToolCallChunk, + type ToolCallMapping, + type ToolCallNormalizationState, +} from './types'; + +export const aggregateToolCalls = ( + toolCalls: NonNullable, +): Array<{ + id?: string; + type?: string; + function: { + arguments: string; + name: string; + }; +}> => { + const aggregated = new Map< + string, + { + order: number; + id?: string; + type?: string; + function: { + arguments: string; + name: string; + }; + } + >(); + const latestKeyByIndex = new Map(); + + toolCalls.forEach((toolCall, position) => { + const normalizedId = createNormalizedToolCallId(toolCall.id, position); + const key = + (toolCall.id ? `id:${normalizedId}` : undefined) ?? + (typeof toolCall.index === 'number' + ? latestKeyByIndex.get(toolCall.index) + : undefined) ?? + `position:${position}`; + const current = aggregated.get(key) ?? { + order: aggregated.size, + function: { + arguments: '', + name: '', + }, + }; + + if (toolCall.id) { + current.id = normalizedId; + } + + if (toolCall.type) { + current.type = toolCall.type; + } + + if (toolCall.function?.name) { + current.function.name += toolCall.function.name; + } + + if (toolCall.function?.arguments) { + current.function.arguments += toolCall.function.arguments; + } + + aggregated.set(key, current); + + if (typeof toolCall.index === 'number') { + latestKeyByIndex.set(toolCall.index, key); + } + }); + + return [...aggregated.values()] + .sort((left, right) => left.order - right.order) + .map(({ order: _order, ...value }, index) => ({ + ...value, + id: value.id ?? createNormalizedToolCallId(undefined, index), + })); +}; + +export const getToolCallStateKey = ( + toolCall: ToolCallChunk, + position: number, +): string => { + if (toolCall.id) { + return `id:${toolCall.id}`; + } + + if (typeof toolCall.index === 'number') { + return `index:${toolCall.index}`; + } + + return `position:${position}`; +}; + +export const createNormalizedToolCallId = ( + sourceId: string | undefined, + normalizedIndex: number, +): string => { + if (sourceId && !sourceId.startsWith('tooluse_')) { + return sourceId; + } + + const suffix = + sourceId?.replace(/^tooluse_/, '') ?? + `${normalizedIndex}_${crypto.randomUUID().replaceAll('-', '')}`; + + return `call_${suffix}`; +}; + +export const resolveToolCallMapping = ( + state: ToolCallNormalizationState, + toolCall: ToolCallChunk, + position: number, +): ToolCallMapping => { + const keys = toolCall.id + ? [`id:${toolCall.id}`] + : [ + typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, + `position:${position}`, + ].filter((value): value is string => value !== null); + const existing = keys + .map((key) => state.mappings.get(key)) + .find((value) => value !== undefined); + + if (existing) { + return existing; + } + + return { + id: createNormalizedToolCallId(toolCall.id, state.nextIndex), + index: state.nextIndex++, + }; +}; + +export const normalizeStreamToolCalls = ( + chunk: ChatStreamChunk, + state: ToolCallNormalizationState, +): ChatStreamChunk => { + if (!chunk.choices?.length) { + return chunk; + } + + return { + ...chunk, + choices: chunk.choices.map((choice) => { + if (!choice.delta?.tool_calls?.length) { + return choice; + } + + return { + ...choice, + delta: { + ...choice.delta, + tool_calls: choice.delta.tool_calls.map((toolCall, position) => { + const mapping = resolveToolCallMapping(state, toolCall, position); + const sourceKey = getToolCallStateKey(toolCall, position); + + state.mappings.set(sourceKey, mapping); + + if (toolCall.id) { + state.mappings.set(`id:${toolCall.id}`, mapping); + } + + if (typeof toolCall.index === 'number') { + state.mappings.set(`index:${toolCall.index}`, mapping); + } + + return { + ...toolCall, + id: mapping.id, + index: mapping.index, + }; + }), + }, + }; + }), + }; +}; + +export const normalizeStreamingResponse = ({ + model, + proxyContext, + route, + upstreamResponse, +}: { + model: string; + proxyContext: ProxyContext; + route: string; + upstreamResponse: Response; +}): Response => { + if (!upstreamResponse.body) { + return createSseResponse(null, { + headers: CORS_HEADERS, + status: upstreamResponse.status, + }); + } + + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const state: ToolCallNormalizationState = { + mappings: new Map(), + nextIndex: 0, + }; + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + const closer = createStreamCloser(); + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + + const stream = new ReadableStream({ + start: (controller) => { + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + let latestUsage: unknown = null; + + const processFrame = (frame: string): string => { + const lines = frame.split('\n'); + const lineIndex = lines.findIndex((line) => line.startsWith('data: ')); + + if (lineIndex === -1) { + return frame; + } + + const raw = lines[lineIndex]?.slice(6).trim() ?? ''; + + if (!raw || raw === '[DONE]') { + return frame; + } + + try { + const chunk = JSON.parse(raw) as ChatStreamChunk; + if (chunk.usage !== undefined) { + latestUsage = chunk.usage; + } + const normalized = normalizeStreamToolCalls(chunk, state); + lines[lineIndex] = `data: ${JSON.stringify(normalized)}`; + return lines.join('\n'); + } catch { + return frame; + } + }; + + const flushFrames = (frames: string[]): void => { + frames.forEach((frame) => { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + return; + } + controller.enqueue(encoder.encode(`${processFrame(frame)}\n\n`)); + }); + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + if (buffer.trim()) { + flushFrames([buffer]); + } + + await recordProxyUsage({ + model, + proxyContext, + route, + usage: latestUsage, + }); + + releaseReader(); + try { + controller.close(); + } catch { + // The downstream stream may have been cancelled while the pump was completing. + } + return; + } + + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + try { + await reader!.cancel(); + } finally { + releaseReader(); + controller.close(); + } + return; + } + flushFrames(frames); + } + }; + + void pump().catch((error) => { + if (cancelled) return; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage !== null) { + closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); + return; + } + + controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + releaseReader(); + } + }, + }); + + return createSseResponse(stream, { + headers: CORS_HEADERS, + status: upstreamResponse.status, + }); +}; + +export const aggregateUpstreamStream = async ( + upstreamResponse: Response, + fallbackModel: string, +): Promise<{ model: string; response: Response; usage: unknown }> => { + const payloadText = await upstreamResponse.text(); + const toolCalls: NonNullable = []; + let responseId = ''; + let responseObject = 'chat.completion'; + let created = Math.floor(Date.now() / 1000); + let model = fallbackModel; + let content = ''; + let reasoningContent = ''; + let finishReason: string | null = 'stop'; + let role = 'assistant'; + let usage: unknown = null; + + for (const frame of payloadText.split('\n\n')) { + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + continue; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + let chunk: ChatStreamChunk; + + try { + chunk = JSON.parse(raw) as ChatStreamChunk; + } catch { + return { + model: fallbackModel, + response: createErrorResponse( + 502, + 'Failed to parse upstream SSE frame', + ), + usage: null, + }; + } + + if (chunk.id) { + responseId = chunk.id; + } + + if (chunk.object) { + responseObject = chunk.object.replace(/\.chunk$/, ''); + } + + if (typeof chunk.created === 'number') { + created = chunk.created; + } + + if (chunk.model) { + model = chunk.model; + } + + if (chunk.usage !== undefined) { + usage = chunk.usage; + } + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (delta?.role) { + role = delta.role; + } + + if (delta?.content) { + content += delta.content; + } + + if (delta?.reasoning_content ?? delta?.reasoning) { + reasoningContent += delta.reasoning_content ?? delta.reasoning; + } + + if (delta?.tool_calls?.length) { + toolCalls.push(...delta.tool_calls); + } + + if (choice?.finish_reason !== undefined) { + finishReason = choice.finish_reason ?? finishReason; + } + } + + const aggregatedToolCalls = aggregateToolCalls(toolCalls); + const message: Record = { + role, + content: content || null, + }; + + if (reasoningContent) { + message.reasoning_content = reasoningContent; + } + + if (aggregatedToolCalls.length) { + message.tool_calls = aggregatedToolCalls; + } + + return { + model, + response: Response.json({ + id: responseId || `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`, + object: responseObject, + created, + model, + choices: [ + { + index: 0, + message, + finish_reason: + finishReason ?? + (aggregatedToolCalls.length ? 'tool_calls' : 'stop'), + }, + ], + usage, + }), + usage, + }; +}; diff --git a/lib/server/proxy/codebuddy/context.ts b/lib/server/proxy/codebuddy/context.ts new file mode 100644 index 0000000..37e48dc --- /dev/null +++ b/lib/server/proxy/codebuddy/context.ts @@ -0,0 +1,159 @@ +import type { NextRequest } from 'next/server'; + +import { resolveRequestAccessKey } from '../auth'; +import { + type CredentialRecord, + findEligibleCredentialRecordByFilename, + findCredentialRecordByFilename, + getCredentialProxySettings, + resolveCredentialForRequest, +} from '../../domain/credentials'; +import { getRequestHeaderMap } from '../../shared/http'; +import type { ProxyContext } from './types'; + +export const getCredentialAffinityKey = ( + request: NextRequest, + accessKeyId: string | null, +): string | undefined => { + const incoming = getRequestHeaderMap(request.headers); + const conversationId = incoming['x-conversation-id']?.trim(); + + if (!conversationId) { + return undefined; + } + + if (accessKeyId) { + return `access-key:${accessKeyId}:conversation:${conversationId}`; + } + + return `global:conversation:${conversationId}`; +}; + +export const getCredentialValue = ( + value: unknown, + candidateKeys: string[], +): string | number | null => { + if (Array.isArray(value)) { + for (const item of value) { + const nested = getCredentialValue(item, candidateKeys); + + if (nested !== null && nested !== '') { + return nested; + } + } + + return null; + } + + if (value && typeof value === 'object') { + for (const key of candidateKeys) { + const direct = (value as Record)[key]; + + if (direct !== undefined && direct !== null && direct !== '') { + return direct as string | number; + } + } + + for (const nestedValue of Object.values(value as Record)) { + const nested = getCredentialValue(nestedValue, candidateKeys); + + if (nested !== null && nested !== '') { + return nested; + } + } + } + + return null; +}; + +export const resolveProxyContext = async ( + request: NextRequest, + model?: string, +): Promise => { + const accessKey = await resolveRequestAccessKey(request); + const credential = await resolveCredentialForRequest({ + accessKeyId: accessKey?.id, + affinityKey: getCredentialAffinityKey(request, accessKey?.id ?? null), + allowedCredentialFilenames: accessKey?.credentialFilenames, + model, + }); + + if (!credential) { + throw new Error('No valid CodeBuddy credentials found'); + } + + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + if (!bearerToken) { + throw new Error('Saved credential does not include a bearer token'); + } + + return { + accessKeyId: accessKey?.id ?? null, + accessKeyName: accessKey?.name ?? null, + auth: { + type: 'bearer', + bearerToken, + userId: String(credential.data.user_id ?? 'unknown'), + credentialData: credential.data, + }, + credentialFilename: credential.filename, + preferences: getCredentialProxySettings(credential.data), + }; +}; + +export const createProxyContextFromCredential = ( + credential: CredentialRecord, +): ProxyContext => { + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + if (!bearerToken) { + throw new Error('Saved credential does not include a bearer token'); + } + + return { + accessKeyId: null, + accessKeyName: null, + auth: { + type: 'bearer', + bearerToken, + userId: String(credential.data.user_id ?? 'unknown'), + credentialData: credential.data, + }, + credentialFilename: credential.filename, + preferences: getCredentialProxySettings(credential.data), + }; +}; + +export const resolveProxyContextByCredentialFilename = async ( + filename: string, + options?: { + accessKey?: { + id?: string | null; + name?: string | null; + }; + allowedCredentialFilenames?: string[]; + requireEligible?: boolean; + }, +): Promise => { + const credential = options?.requireEligible + ? await findEligibleCredentialRecordByFilename( + filename, + options.allowedCredentialFilenames, + ) + : await findCredentialRecordByFilename(filename); + + if (!credential) { + throw new Error('Selected credential was not found'); + } + + return { + ...createProxyContextFromCredential(credential), + accessKeyId: options?.accessKey?.id ?? null, + accessKeyName: options?.accessKey?.name ?? null, + }; +}; diff --git a/lib/server/proxy/codebuddy/models.ts b/lib/server/proxy/codebuddy/models.ts new file mode 100644 index 0000000..06ca2ef --- /dev/null +++ b/lib/server/proxy/codebuddy/models.ts @@ -0,0 +1,242 @@ +import type { NextRequest } from 'next/server'; + +import { resolveRequestAccessKey } from '../auth'; +import { getCodeBuddyApiEndpoint } from '../../domain/config'; +import { + type CredentialData, + type CredentialRecord, + getCredentialSupportedModels, + listEligibleCredentialRecords, +} from '../../domain/credentials'; +import { getCredentialValue } from './context'; +import type { DiscoveredModel } from './types'; + +export const getModelsForCredential = async ({ + bearerToken, + credentialData, +}: { + bearerToken: string; + credentialData: CredentialData; +}): Promise => { + const configuredEndpoint = await getCodeBuddyApiEndpoint(); + const headers = new Headers({ + Accept: 'application/json', + Authorization: `Bearer ${bearerToken}`, + 'X-Product': 'SaaS', + }); + const domain = getCredentialValue(credentialData, ['domain']); + const apiEndpoint = String(domain ?? '') + .toLowerCase() + .endsWith('workbuddy.ai') + ? 'https://www.workbuddy.ai' + : configuredEndpoint; + const enterpriseId = getCredentialValue(credentialData, [ + 'enterprise_id', + 'enterpriseId', + ]); + const tenantId = + getCredentialValue(credentialData, ['tenant_id', 'tenantId']) ?? + enterpriseId; + const userId = getCredentialValue(credentialData, ['user_id', 'userId']); + + if (domain) { + headers.set('X-Domain', String(domain)); + } + + if (enterpriseId) { + headers.set('X-Enterprise-Id', String(enterpriseId)); + } + + if (tenantId) { + headers.set('X-Tenant-Id', String(tenantId)); + } + if (userId) { + headers.set('X-User-Id', String(userId)); + } + + const fetchModels = async (path: string): Promise => + fetch(new URL(path, apiEndpoint), { + headers, + signal: AbortSignal.timeout(15_000), + }); + let response = await fetchModels('/v3/config'); + + if ([400, 404, 405].includes(response.status)) { + // Upstream splits this route by account scope: enterprise accounts must hit + // their own segment, otherwise they are served the personal model catalog. + const enterpriseScope = String(enterpriseId ?? '').trim() || 'personal'; + response = await fetchModels( + `/console/enterprises/${encodeURIComponent(enterpriseScope)}/models`, + ); + } + + if (!response.ok) { + throw new Error(`Model discovery failed with status ${response.status}`); + } + + const payload = (await response.json()) as { + code?: unknown; + data?: { + agents?: Array<{ models?: unknown; name?: unknown }>; + models?: Array<{ disabled?: unknown; id?: unknown; name?: unknown }>; + }; + }; + + if (payload.code !== 0) { + throw new Error('Model discovery returned an unsuccessful response'); + } + + const cliModels = payload.data?.agents?.find( + (agent) => agent.name === 'cli', + )?.models; + const modelsById = new Map( + (payload.data?.models ?? []).flatMap((model) => { + const id = typeof model.id === 'string' ? model.id.trim() : ''; + + if (!id || model.disabled === true) { + return []; + } + + return [ + [ + id, + { + displayName: + typeof model.name === 'string' && model.name.trim() + ? model.name + : id, + id, + }, + ] as const, + ]; + }), + ); + const declaredModelIds = new Set( + (payload.data?.models ?? []) + .map((model) => (typeof model.id === 'string' ? model.id.trim() : '')) + .filter(Boolean), + ); + + if (!Array.isArray(cliModels)) { + return []; + } + + return cliModels.flatMap((modelId) => { + if (typeof modelId !== 'string') { + return []; + } + + const model = modelsById.get(modelId); + if (!model && declaredModelIds.has(modelId)) { + return []; + } + return [ + model ?? { + displayName: modelId, + id: modelId, + }, + ]; + }); +}; + +export const getModelsForCredentials = async ( + credentials: CredentialRecord[], +): Promise => { + const settled = await Promise.allSettled( + credentials.map((credential) => { + const supportedModels = getCredentialSupportedModels(credential.data); + + if (supportedModels.length) { + return Promise.resolve( + supportedModels.map((id) => ({ displayName: id, id })), + ); + } + + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + return bearerToken + ? getModelsForCredential({ + bearerToken, + credentialData: credential.data, + }) + : Promise.resolve([]); + }), + ); + const models = new Map(); + + settled.forEach((result) => { + if (result.status !== 'fulfilled') { + return; + } + + result.value.forEach((model) => { + models.set(model.id, model); + }); + }); + + return [...models.values()].sort((left, right) => + left.id.localeCompare(right.id), + ); +}; + +export const getModelsByCredential = async ( + credentials: CredentialRecord[], +): Promise< + Record +> => { + const results = await Promise.all( + credentials.map(async (credential) => { + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + try { + const models = bearerToken + ? await getModelsForCredential({ + bearerToken, + credentialData: credential.data, + }) + : []; + + return [credential.filename, { error: null, models }] as const; + } catch (error) { + return [ + credential.filename, + { + error: + error instanceof Error ? error.message : 'Model discovery failed', + models: [], + }, + ] as const; + } + }), + ); + + return Object.fromEntries(results); +}; + +export const getModelsResponse = async ( + request?: NextRequest, +): Promise => { + const accessKey = request ? await resolveRequestAccessKey(request) : null; + const models = ( + await getModelsForCredentials( + await listEligibleCredentialRecords(accessKey?.credentialFilenames), + ) + ).map((model) => ({ + id: model.id, + slug: model.id, + display_name: model.displayName, + object: 'model', + created: 0, + owned_by: 'codebuddy', + })); + + return Response.json({ + object: 'list', + data: models, + models, + }); +}; diff --git a/lib/server/proxy/codebuddy/responses-request.ts b/lib/server/proxy/codebuddy/responses-request.ts new file mode 100644 index 0000000..bd6cc96 --- /dev/null +++ b/lib/server/proxy/codebuddy/responses-request.ts @@ -0,0 +1,382 @@ +import { stringifyContent } from '../../shared/content'; +import { resolveHyResponsesReasoning } from '../../shared/hy-thought-depth'; +import type { ChatRequestBody } from './types'; + +export const isImageContentPart = (part: unknown): boolean => { + if (!part || typeof part !== 'object') { + return false; + } + + const value = part as { image_url?: unknown; type?: unknown }; + + if (value.type === 'image_url' || value.type === 'input_image') { + return true; + } + + // Accept the shapes an OpenAI-compatible client may send even when `type` + // is absent or unexpected: any part carrying an image URL is an image. + return ( + typeof value.image_url === 'string' || + Boolean( + value.image_url && + typeof value.image_url === 'object' && + typeof (value.image_url as { url?: unknown }).url === 'string', + ) + ); +}; + +/** + * Reads the image URL out of a Responses `input_image` / `image_url` part. + * Returns undefined when the part carries no usable URL, so callers can drop it + * rather than forwarding a block the upstream would reject. + */ +export const extractImageUrl = (part: unknown): string | undefined => { + if (!part || typeof part !== 'object') { + return undefined; + } + + const { image_url: imageUrl } = part as { image_url?: unknown }; + + // `input_image` carries a bare URL string; the OpenAI Chat-style + // `image_url` part nests it under `url`. + if (typeof imageUrl === 'string') { + return imageUrl || undefined; + } + + if ( + imageUrl && + typeof imageUrl === 'object' && + typeof (imageUrl as { url?: unknown }).url === 'string' + ) { + return (imageUrl as { url: string }).url || undefined; + } + + return undefined; +}; + +export const mapChatContentToResponses = ( + content: unknown, +): Array> => { + if (!Array.isArray(content)) { + return [ + { + text: stringifyContent(content), + type: 'input_text', + }, + ]; + } + + return content.flatMap((part): Array> => { + if (typeof part === 'string') { + return [{ text: part, type: 'input_text' }]; + } + if (!part || typeof part !== 'object') { + return [{ text: JSON.stringify(part), type: 'input_text' }]; + } + const value = part as { + image_url?: string | { detail?: unknown; url?: unknown }; + text?: unknown; + type?: unknown; + }; + if (value.type === 'image_url') { + const imageUrl = + typeof value.image_url === 'string' + ? value.image_url + : value.image_url?.url; + if (typeof imageUrl === 'string' && imageUrl) { + const detail = + typeof value.image_url === 'object' && + typeof value.image_url.detail === 'string' + ? value.image_url.detail + : undefined; + return [ + { + image_url: imageUrl, + ...(detail ? { detail } : {}), + type: 'input_image', + }, + ]; + } + } + if (value.type === 'input_image' && typeof value.image_url === 'string') { + return [{ image_url: value.image_url, type: 'input_image' }]; + } + if (typeof value.text === 'string') { + return [{ text: value.text, type: 'input_text' }]; + } + return [{ text: JSON.stringify(value), type: 'input_text' }]; + }); +}; + +export const translateChatToolChoiceToResponses = ( + toolChoice: unknown, +): unknown => { + if (typeof toolChoice === 'string') return toolChoice; + if (!toolChoice || typeof toolChoice !== 'object') return undefined; + const value = toolChoice as { + function?: { name?: unknown }; + name?: unknown; + type?: unknown; + }; + if (value.type !== 'function') return toolChoice; + const name = value.function?.name ?? value.name; + return typeof name === 'string' ? { name, type: 'function' } : toolChoice; +}; + +export const translateChatResponseFormatToResponses = ( + responseFormat: unknown, +): Record | undefined => { + if (!responseFormat || typeof responseFormat !== 'object') return undefined; + const value = responseFormat as { + json_schema?: Record; + type?: unknown; + }; + if (value.type === 'json_object') { + return { format: { type: 'json_object' } }; + } + if (value.type !== 'json_schema' || !value.json_schema) return undefined; + const schema = value.json_schema; + if (typeof schema.name !== 'string' || !schema.name) return undefined; + return { + format: { + ...(schema.description ? { description: schema.description } : {}), + name: schema.name, + schema: schema.schema ?? { type: 'object', properties: {} }, + ...(typeof schema.strict === 'boolean' ? { strict: schema.strict } : {}), + type: 'json_schema', + }, + }; +}; + +export const translateChatThinkingToResponses = ( + thinking: Record | undefined, + reasoningEffort: string | undefined, +): Record | undefined => { + if (!thinking) + return reasoningEffort ? { effort: reasoningEffort } : undefined; + + if (thinking.type === 'disabled') return { effort: 'none' }; + if (thinking.type !== 'adaptive' && thinking.type !== 'enabled') { + return undefined; + } + + const budgetTokens = + typeof thinking.budget_tokens === 'number' + ? thinking.budget_tokens + : Number.NaN; + const effort = reasoningEffort + ? reasoningEffort + : Number.isFinite(budgetTokens) + ? budgetTokens <= 2_048 + ? 'low' + : budgetTokens <= 8_192 + ? 'medium' + : 'high' + : undefined; + + return { + ...(effort ? { effort } : {}), + summary: 'auto', + }; +}; + +/** + * Codex sends `reasoning.effort` in the OpenAI vocabulary, which Hy models do + * not accept, so the effort is rewritten onto the Hy vocabulary before the body + * is forwarded. + */ +export const resolveHyResponsesBody = async ( + body: Record, +): Promise> => { + const reasoning = await resolveHyResponsesReasoning( + typeof body.model === 'string' ? body.model : undefined, + body.reasoning as Record | undefined, + ); + + return { ...body, reasoning }; +}; + +export const normalizeResponsesUpstreamBody = async ( + body: Record, +): Promise> => { + const { messages, ...rest } = body; + + if (rest.input !== undefined || !Array.isArray(messages)) { + return resolveHyResponsesBody(rest); + } + + const systemInstructions = messages + .filter((message) => { + return ( + message && + typeof message === 'object' && + ((message as { role?: unknown }).role === 'system' || + (message as { role?: unknown }).role === 'developer') + ); + }) + .map((message) => { + return stringifyContent((message as { content?: unknown }).content); + }) + .filter(Boolean) + .join('\n\n'); + const input = messages.flatMap((message) => { + if (!message || typeof message !== 'object') return []; + const value = message as { content?: unknown; role?: unknown }; + if (value.role === 'system' || value.role === 'developer') return []; + const role = value.role === 'assistant' ? 'assistant' : 'user'; + return [ + { + content: mapChatContentToResponses(value.content), + role, + }, + ]; + }); + + const existingInstructions = + typeof rest.instructions === 'string' ? rest.instructions.trim() : ''; + const instructions = [existingInstructions, systemInstructions] + .filter(Boolean) + .join('\n\n'); + + return resolveHyResponsesBody({ + ...rest, + ...(instructions ? { instructions } : {}), + input, + }); +}; + +export const buildResponsesBodyFromChat = async ( + body: ChatRequestBody, +): Promise> => { + const instructions = body.messages + ?.filter( + (message) => message.role === 'system' || message.role === 'developer', + ) + .map((message) => stringifyContent(message.content)) + .filter(Boolean) + .join('\n\n'); + const input = + body.messages + ?.filter( + (message) => message.role !== 'system' && message.role !== 'developer', + ) + .map((message) => { + if (message.role === 'tool') { + // A tool may return an image, e.g. a screenshot. The upstream + // `function_call_output` carries `output` as structured content, so + // an image part is preserved there; stringifying it would hand the + // model a base64 dump instead of the image. + const toolOutput = Array.isArray(message.content) + ? message.content.filter( + (part) => part !== null && part !== undefined, + ) + : message.content; + const hasImage = Array.isArray(toolOutput) + ? toolOutput.some(isImageContentPart) + : isImageContentPart(toolOutput); + + return { + call_id: message.tool_call_id, + output: hasImage + ? mapChatContentToResponses(toolOutput) + : stringifyContent(toolOutput), + type: 'function_call_output', + }; + } + const toolCalls = Array.isArray(message.tool_calls) + ? message.tool_calls + : []; + const functionCalls = toolCalls.flatMap((toolCall) => { + if (!toolCall || typeof toolCall !== 'object') return []; + const call = toolCall as { + function?: { arguments?: unknown; name?: unknown }; + id?: unknown; + }; + if (typeof call.function?.name !== 'string') return []; + return [ + { + arguments: String(call.function.arguments ?? ''), + call_id: String(call.id ?? crypto.randomUUID()), + name: call.function.name, + type: 'function_call', + }, + ]; + }); + const content = mapChatContentToResponses(message.content); + const hasContent = content.some((part) => { + return ( + (part.type === 'input_text' && Boolean(part.text)) || + (part.type === 'input_image' && Boolean(part.image_url)) + ); + }); + const shouldOmitMessage = + message.role === 'assistant' && + functionCalls.length > 0 && + !hasContent; + + return [ + ...(shouldOmitMessage + ? [] + : [ + { + content, + role: message.role === 'assistant' ? 'assistant' : 'user', + }, + ]), + ...functionCalls, + ]; + }) + .flat() ?? []; + const tools = body.tools?.flatMap((tool) => { + if (!tool || typeof tool !== 'object') return []; + const value = tool as { + function?: Record; + type?: unknown; + }; + const definition: Record = + value.type === 'function' && value.function ? value.function : value; + if (typeof definition.name !== 'string') return []; + return [ + { + ...definition, + parameters: definition.parameters ?? { type: 'object', properties: {} }, + type: 'function', + }, + ]; + }); + const text = translateChatResponseFormatToResponses(body.response_format); + const reasoning = await resolveHyResponsesReasoning( + body.model, + translateChatThinkingToResponses(body.thinking, body.reasoning_effort), + ); + + return { + ...(instructions ? { instructions } : {}), + input, + max_output_tokens: body.max_tokens ?? body.max_completion_tokens, + model: body.model, + parallel_tool_calls: body.parallel_tool_calls, + reasoning, + stream: Boolean(body.stream), + temperature: body.temperature, + top_p: body.top_p, + ...(tools?.length ? { tools } : {}), + ...(body.tool_choice + ? { tool_choice: translateChatToolChoiceToResponses(body.tool_choice) } + : {}), + ...(text ? { text } : {}), + }; +}; + +export const getUnsupportedResponsesChatOptions = ( + body: ChatRequestBody, +): string[] => { + return [ + body.frequency_penalty !== undefined ? 'frequency_penalty' : null, + body.presence_penalty !== undefined ? 'presence_penalty' : null, + body.thinking !== undefined && + !translateChatThinkingToResponses(body.thinking, body.reasoning_effort) + ? 'thinking' + : null, + ].filter((name): name is string => Boolean(name)); +}; diff --git a/lib/server/proxy/codebuddy/responses-response.ts b/lib/server/proxy/codebuddy/responses-response.ts new file mode 100644 index 0000000..76c791d --- /dev/null +++ b/lib/server/proxy/codebuddy/responses-response.ts @@ -0,0 +1,700 @@ +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; +import { + chatStreamErrorChunks, + createStreamCloser, + readTimeoutFrame, + toUpstreamTimeoutMessage, +} from '../../shared/upstream-timeout'; +import { + extractResponsesUsage, + mapResponsesUsageToChat, + parseUsageHeader, + recordProxyUsage, +} from './usage'; +import { + CORS_HEADERS, + MAX_STREAM_FRAME_LENGTH, + type ProxyContext, +} from './types'; + +export const normalizeStopSequences = ( + stop: string | string[] | undefined, +): string[] => { + return (Array.isArray(stop) ? stop : stop ? [stop] : []).filter(Boolean); +}; + +export const findFirstStopSequence = ( + text: string, + stopSequences: string[], +): number | null => { + return stopSequences.reduce((earliest, stopSequence) => { + const index = text.indexOf(stopSequence); + if (index < 0) return earliest; + return earliest === null ? index : Math.min(earliest, index); + }, null); +}; + +export const getPendingStopPrefixLength = ( + text: string, + stopSequences: string[], +): number => { + const maximumLength = Math.min( + text.length, + Math.max( + 0, + ...stopSequences.map((stopSequence) => stopSequence.length - 1), + ), + ); + + for (let length = maximumLength; length > 0; length -= 1) { + const suffix = text.slice(-length); + if (stopSequences.some((stopSequence) => stopSequence.startsWith(suffix))) { + return length; + } + } + + return 0; +}; + +export const extractResponsesReasoningText = (output: unknown[]): string => { + return output + .flatMap((item) => { + if (!item || typeof item !== 'object') return []; + const value = item as { + content?: unknown; + summary?: unknown; + type?: unknown; + }; + if (value.type !== 'reasoning') return []; + return [value.summary, value.content].flatMap((parts) => { + if (!Array.isArray(parts)) return []; + return parts.flatMap((part) => { + if (!part || typeof part !== 'object') return []; + const text = (part as { text?: unknown }).text; + return typeof text === 'string' ? [text] : []; + }); + }); + }) + .join(''); +}; + +export const mapResponsesPayloadToChat = ( + payload: Record, + model: string, + stop: string | string[] | undefined, +): Record => { + const output = Array.isArray(payload.output) ? payload.output : []; + const toolCalls = output.flatMap((item) => { + if (!item || typeof item !== 'object') return []; + const value = item as Record; + if ( + value.type !== 'function_call' && + value.type !== 'mcp_call' && + value.type !== 'custom_tool_call' + ) { + return []; + } + const isCustomToolCall = value.type === 'custom_tool_call'; + const customArguments = JSON.stringify({ + input: String(value.input ?? value.arguments ?? ''), + }); + return [ + { + function: { + arguments: String( + isCustomToolCall ? customArguments : (value.arguments ?? ''), + ), + name: String(value.name ?? 'function'), + }, + id: String(value.call_id ?? value.id ?? crypto.randomUUID()), + type: 'function', + }, + ]; + }); + const usage = + payload.usage && typeof payload.usage === 'object' + ? (payload.usage as Record) + : undefined; + const inputTokens = Number(usage?.input_tokens ?? 0); + const outputTokens = Number(usage?.output_tokens ?? 0); + + const rawOutputText = + typeof payload.output_text === 'string' + ? payload.output_text + : output + .flatMap((item) => { + if (!item || typeof item !== 'object') return []; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) return []; + return content.flatMap((part) => { + if (!part || typeof part !== 'object') return []; + const value = part as { text?: unknown; type?: unknown }; + return value.type === 'output_text' && + typeof value.text === 'string' + ? [value.text] + : []; + }); + }) + .join(''); + const stopIndex = findFirstStopSequence( + rawOutputText, + normalizeStopSequences(stop), + ); + const outputText = + stopIndex === null ? rawOutputText : rawOutputText.slice(0, stopIndex); + const reasoningText = extractResponsesReasoningText(output); + const incompleteReason = + payload.incomplete_details && typeof payload.incomplete_details === 'object' + ? (payload.incomplete_details as { reason?: unknown }).reason + : undefined; + const finishReason = + payload.status === 'incomplete' + ? incompleteReason === 'content_filter' + ? 'content_filter' + : 'length' + : toolCalls.length + ? 'tool_calls' + : 'stop'; + + return { + choices: [ + { + finish_reason: finishReason, + index: 0, + message: { + content: outputText || null, + role: 'assistant', + ...(reasoningText ? { reasoning_content: reasoningText } : {}), + ...(toolCalls.length ? { tool_calls: toolCalls } : {}), + }, + }, + ], + created: Number(payload.created_at ?? Math.floor(Date.now() / 1000)), + id: String(payload.id ?? `chatcmpl-${crypto.randomUUID()}`), + model, + object: 'chat.completion', + usage: { + completion_tokens: outputTokens, + prompt_tokens: inputTokens, + total_tokens: Number(usage?.total_tokens ?? inputTokens + outputTokens), + }, + }; +}; + +export const mapResponsesStreamToChat = ( + upstreamResponse: Response, + model: string, + proxyContext: ProxyContext, + route: string, + stop: string | string[] | undefined, + includeUsage: boolean, +): Response => { + const closer = createStreamCloser(); + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const responseId = `chatcmpl-${crypto.randomUUID()}`; + let reader: ReadableStreamDefaultReader | null = + upstreamResponse.body?.getReader() ?? null; + const fallbackUsage = parseUsageHeader(upstreamResponse); + let buffer = ''; + let emittedFinish = false; + let emittedUsage = false; + let hasToolCalls = false; + let latestUsage = fallbackUsage; + let usageRecorded = false; + const stopSequences = normalizeStopSequences(stop); + let pendingStopText = ''; + const toolIndexes = new Map(); + const toolCallIds = new Map(); + const customToolCallIds = new Set(); + const closedCustomToolCallIds = new Set(); + let nextToolIndex = 0; + let stoppedLocally = false; + + const getToolIndex = (itemId: string): number => { + const existing = toolIndexes.get(itemId); + if (existing !== undefined) return existing; + const index = nextToolIndex; + nextToolIndex += 1; + toolIndexes.set(itemId, index); + return index; + }; + + const encodeChunk = (choice: Record): Uint8Array => { + return encoder.encode( + `data: ${JSON.stringify({ + choices: [choice], + created: Math.floor(Date.now() / 1000), + id: responseId, + model, + object: 'chat.completion.chunk', + })}\n\n`, + ); + }; + + const enqueueUsage = ( + controller: ReadableStreamDefaultController, + ): void => { + if (!includeUsage || emittedUsage) return; + const usage = mapResponsesUsageToChat(latestUsage); + if (!usage) return; + + emittedUsage = true; + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [], + created: Math.floor(Date.now() / 1000), + id: responseId, + model, + object: 'chat.completion.chunk', + usage, + })}\n\n`, + ), + ); + }; + + const recordStreamUsage = async (): Promise => { + if (usageRecorded) return; + usageRecorded = true; + try { + await recordProxyUsage({ + model, + proxyContext, + route, + usage: latestUsage, + }); + } catch (error) { + console.error('[CodeBuddy2API] Failed to record Responses stream usage', { + error, + route, + }); + } + }; + + const cancelAndReleaseReader = async (reason?: unknown): Promise => { + try { + await reader?.cancel(reason); + } catch (error) { + console.error('[CodeBuddy2API] Failed to cancel Responses stream', { + error, + route, + }); + } finally { + reader?.releaseLock(); + reader = null; + } + }; + + const emitCustomToolCallClosures = ( + controller: ReadableStreamDefaultController, + ): void => { + customToolCallIds.forEach((itemId) => { + if (closedCustomToolCallIds.has(itemId)) return; + const index = getToolIndex(itemId); + const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; + controller.enqueue( + encodeChunk({ + delta: { + tool_calls: [{ function: { arguments: '"}' }, id: callId, index }], + }, + index: 0, + }), + ); + closedCustomToolCallIds.add(itemId); + }); + }; + + const stream = new ReadableStream({ + async pull(controller) { + if (!reader) { + await recordStreamUsage(); + controller.close(); + return; + } + while (true) { + let readResult: ReadableStreamReadResult; + try { + readResult = await reader.read(); + } catch (error) { + await recordStreamUsage(); + reader.releaseLock(); + reader = null; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage !== null) { + closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); + return; + } + + controller.error(error); + return; + } + const { done, value } = readResult; + if (done) { + if (pendingStopText) { + controller.enqueue( + encodeChunk({ + delta: { content: pendingStopText }, + index: 0, + }), + ); + pendingStopText = ''; + } + emitCustomToolCallClosures(controller); + if (!emittedFinish) { + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: hasToolCalls ? 'tool_calls' : 'stop', + index: 0, + }), + ); + } + enqueueUsage(controller); + controller.enqueue(encodeDoneFrame()); + await recordStreamUsage(); + reader.releaseLock(); + reader = null; + controller.close(); + return; + } + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split(/\r?\n\r?\n/); + buffer = frames.pop() ?? ''; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader(); + await recordStreamUsage(); + controller.close(); + return; + } + let emitted = false; + for (const frame of frames) { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader(); + await recordStreamUsage(); + controller.close(); + return; + } + const dataLine = frame + .split(/\r?\n/) + .find((line) => line.startsWith('data: ')); + if (!dataLine || dataLine === 'data: [DONE]') continue; + // The upstream here is the chat pipeline, which reports a deadline as + // a terminal error chunk and closes cleanly. Without this the failure + // would be reported to the client as an empty successful response. + const upstreamError = readTimeoutFrame(frame); + + if (upstreamError !== null) { + closer.fail(controller, chatStreamErrorChunks(upstreamError)); + await cancelAndReleaseReader(); + await recordStreamUsage(); + return; + } + try { + const event = JSON.parse(dataLine.slice(6)) as { + delta?: unknown; + item?: unknown; + item_id?: unknown; + output_index?: unknown; + error?: unknown; + response?: unknown; + type?: unknown; + }; + latestUsage = extractResponsesUsage(event) ?? latestUsage; + if ( + stoppedLocally && + event.type !== 'response.completed' && + event.type !== 'response.incomplete' + ) { + continue; + } + if (event.type === 'response.output_text.delta') { + const delta = String(event.delta ?? ''); + if (stopSequences.length) { + pendingStopText += delta; + const stopIndex = findFirstStopSequence( + pendingStopText, + stopSequences, + ); + if (stopIndex !== null) { + const content = pendingStopText.slice(0, stopIndex); + if (content) { + controller.enqueue( + encodeChunk({ delta: { content }, index: 0 }), + ); + } + pendingStopText = ''; + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: 'stop', + index: 0, + }), + ); + emittedFinish = true; + stoppedLocally = true; + emitted = true; + continue; + } + + const pendingLength = getPendingStopPrefixLength( + pendingStopText, + stopSequences, + ); + const content = pendingStopText.slice( + 0, + pendingStopText.length - pendingLength, + ); + pendingStopText = pendingLength + ? pendingStopText.slice(-pendingLength) + : ''; + if (!content) continue; + controller.enqueue( + encodeChunk({ delta: { content }, index: 0 }), + ); + emitted = true; + continue; + } + controller.enqueue( + encodeChunk({ + delta: { content: delta }, + index: 0, + }), + ); + emitted = true; + continue; + } + if ( + event.type === 'response.reasoning_summary_text.delta' || + event.type === 'response.reasoning_text.delta' + ) { + controller.enqueue( + encodeChunk({ + delta: { reasoning_content: String(event.delta ?? '') }, + index: 0, + }), + ); + emitted = true; + continue; + } + if ( + event.type === 'response.output_item.added' && + event.item && + typeof event.item === 'object' + ) { + const item = event.item as { + arguments?: unknown; + call_id?: unknown; + id?: unknown; + input?: unknown; + name?: unknown; + type?: unknown; + }; + if ( + item.type !== 'function_call' && + item.type !== 'mcp_call' && + item.type !== 'custom_tool_call' + ) { + continue; + } + const isCustomToolCall = item.type === 'custom_tool_call'; + const initialArguments = isCustomToolCall + ? `{"input":"${JSON.stringify( + String(item.input ?? item.arguments ?? ''), + ).slice(1, -1)}` + : String(item.arguments ?? ''); + const itemId = String(item.id ?? item.call_id ?? nextToolIndex); + const index = getToolIndex(itemId); + const callId = String(item.call_id ?? item.id ?? itemId); + toolCallIds.set(itemId, callId); + if (isCustomToolCall) { + customToolCallIds.add(itemId); + } + hasToolCalls = true; + controller.enqueue( + encodeChunk({ + delta: { + tool_calls: [ + { + function: { + arguments: initialArguments, + name: String(item.name ?? 'function'), + }, + id: callId, + index, + type: 'function', + }, + ], + }, + index: 0, + }), + ); + emitted = true; + continue; + } + if ( + event.type === 'response.function_call_arguments.delta' || + event.type === 'response.mcp_call_arguments.delta' || + event.type === 'response.custom_tool_call_input.delta' + ) { + const itemId = String( + event.item_id ?? event.output_index ?? nextToolIndex, + ); + const index = getToolIndex(itemId); + const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; + toolCallIds.set(itemId, callId); + hasToolCalls = true; + const argumentDelta = + event.type === 'response.custom_tool_call_input.delta' + ? JSON.stringify(String(event.delta ?? '')).slice(1, -1) + : String(event.delta ?? ''); + controller.enqueue( + encodeChunk({ + delta: { + tool_calls: [ + { + function: { arguments: argumentDelta }, + id: callId, + index, + }, + ], + }, + index: 0, + }), + ); + emitted = true; + continue; + } + if (event.type === 'response.completed') { + if (pendingStopText) { + controller.enqueue( + encodeChunk({ + delta: { content: pendingStopText }, + index: 0, + }), + ); + pendingStopText = ''; + } + emitCustomToolCallClosures(controller); + if (!emittedFinish) { + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: hasToolCalls ? 'tool_calls' : 'stop', + index: 0, + }), + ); + } + enqueueUsage(controller); + emittedFinish = true; + emitted = true; + if (stoppedLocally) { + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader('Stop sequence matched'); + await recordStreamUsage(); + controller.close(); + return; + } + continue; + } + if (event.type === 'response.incomplete') { + if (pendingStopText) { + controller.enqueue( + encodeChunk({ + delta: { content: pendingStopText }, + index: 0, + }), + ); + pendingStopText = ''; + } + emitCustomToolCallClosures(controller); + const incompleteReason = + event.response && typeof event.response === 'object' + ? ( + (event.response as { incomplete_details?: unknown }) + .incomplete_details as { reason?: unknown } | undefined + )?.reason + : undefined; + if (!emittedFinish) { + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: + incompleteReason === 'content_filter' + ? 'content_filter' + : 'length', + index: 0, + }), + ); + } + enqueueUsage(controller); + emittedFinish = true; + emitted = true; + if (stoppedLocally) { + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader('Stop sequence matched'); + await recordStreamUsage(); + controller.close(); + return; + } + continue; + } + if ( + event.type === 'response.failed' || + event.type === 'response.error' || + event.type === 'error' + ) { + const failure = + event.error ?? + (event.response && typeof event.response === 'object' + ? (event.response as { error?: unknown }).error + : undefined); + const message = + failure && typeof failure === 'object' + ? String( + (failure as { message?: unknown }).message ?? failure, + ) + : String(failure ?? 'Upstream Responses stream failed'); + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ error: { message } })}\n\n`, + ), + ); + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader(); + await recordStreamUsage(); + controller.close(); + return; + } + } catch { + // Ignore malformed upstream events and continue reading. + } + } + if (stoppedLocally) continue; + if (emitted) return; + } + }, + async cancel(reason) { + await cancelAndReleaseReader(reason); + await recordStreamUsage(); + }, + }); + + return createSseResponse(stream, { + headers: CORS_HEADERS, + status: upstreamResponse.status, + }); +}; diff --git a/lib/server/proxy/codebuddy/server-tools.ts b/lib/server/proxy/codebuddy/server-tools.ts new file mode 100644 index 0000000..a8e6b66 --- /dev/null +++ b/lib/server/proxy/codebuddy/server-tools.ts @@ -0,0 +1,239 @@ +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../../search/tool'; +import { aggregateUpstreamStream } from './chat-stream'; +import { type ChatStreamChunk, type StreamProbeState } from './types'; + +export const isServerWebToolName = (name: string): boolean => { + const normalized = normalizeToolName(name); + + return ( + normalized === normalizeToolName(WEB_SEARCH_TOOL_NAME) || + normalized === normalizeToolName(WEB_FETCH_TOOL_NAME) + ); +}; + +export const mergeToolName = (previous: string, incoming: string): string => { + if (!previous || incoming.startsWith(previous)) { + return incoming; + } + + if (!incoming || previous.endsWith(incoming)) { + return previous; + } + + return previous + incoming; +}; + +export const classifyStreamFrame = ( + frame: string, + state: StreamProbeState, + serverToolNames: string[], +): 'passthrough' | 'server-tool' | null => { + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + return null; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + return raw === '[DONE]' ? 'passthrough' : null; + } + + try { + const chunk = JSON.parse(raw) as ChatStreamChunk; + + for (const choice of chunk.choices ?? []) { + const delta = choice.delta; + const toolCalls = delta?.tool_calls ?? []; + let hasNonServerTool = false; + + for (const [position, toolCall] of toolCalls.entries()) { + const incoming = toolCall.function?.name; + + if (typeof incoming !== 'string' || !incoming) { + continue; + } + + const key = + typeof toolCall.index === 'number' + ? `index:${toolCall.index}` + : toolCall.id + ? `id:${toolCall.id}` + : `position:${position}`; + const name = mergeToolName(state.toolNames.get(key) ?? '', incoming); + const normalized = normalizeToolName(name); + + state.toolNames.set(key, name); + + if (isServerWebToolName(name) && serverToolNames.includes(normalized)) { + return 'server-tool'; + } + + if ( + normalized && + !serverToolNames.some((serverName) => + serverName.startsWith(normalized), + ) + ) { + hasNonServerTool = true; + } + } + + if ( + delta?.content || + delta?.reasoning_content || + delta?.reasoning || + hasNonServerTool || + choice.finish_reason != null + ) { + return 'passthrough'; + } + } + } catch { + return null; + } + + return null; +}; + +export const concatenateChunks = (chunks: Uint8Array[]): ArrayBuffer => { + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const combined = new Uint8Array(size); + let offset = 0; + + for (const chunk of chunks) { + combined.set(chunk, offset); + offset += chunk.byteLength; + } + + return combined.buffer; +}; + +export const createResponseWithBody = ( + body: BodyInit, + response: Response, +): Response => + new Response(body, { + headers: response.headers, + status: response.status, + statusText: response.statusText, + }); + +export const createReplayStreamResponse = ({ + chunks, + reader, + response, +}: { + chunks: Uint8Array[]; + reader: ReadableStreamDefaultReader; + response: Response; +}): Response => { + let cancelled = false; + + const stream = new ReadableStream({ + start: (controller) => { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + + const pump = async (): Promise => { + while (true) { + const { done, value } = await reader.read(); + + if (cancelled) { + return; + } + + if (done) { + reader.releaseLock(); + controller.close(); + return; + } + + controller.enqueue(value); + } + }; + + void pump().catch((error) => { + if (!cancelled) { + controller.error(error); + } + }); + }, + async cancel(reason): Promise { + cancelled = true; + try { + await reader.cancel(reason); + } finally { + reader.releaseLock(); + } + }, + }); + + return createResponseWithBody(stream, response); +}; + +export const detectServerToolStream = async ( + response: Response, + fallbackModel: string, + serverToolNames: string[], +): Promise => { + if (!response.body) { + return response; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const chunks: Uint8Array[] = []; + const state: StreamProbeState = { toolNames: new Map() }; + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + reader.releaseLock(); + return createResponseWithBody(concatenateChunks(chunks), response); + } + + chunks.push(value); + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop() ?? ''; + + for (const frame of frames) { + const classification = classifyStreamFrame(frame, state, serverToolNames); + + if (classification === 'passthrough') { + return createReplayStreamResponse({ chunks, reader, response }); + } + + if (classification === 'server-tool') { + while (true) { + const remainder = await reader.read(); + + if (remainder.done) { + reader.releaseLock(); + break; + } + + chunks.push(remainder.value); + } + + return ( + await aggregateUpstreamStream( + new Response(concatenateChunks(chunks)), + fallbackModel, + ) + ).response; + } + } + } +}; diff --git a/lib/server/proxy/codebuddy/types.ts b/lib/server/proxy/codebuddy/types.ts new file mode 100644 index 0000000..c79a1a9 --- /dev/null +++ b/lib/server/proxy/codebuddy/types.ts @@ -0,0 +1,137 @@ +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../../search/tool'; + +/** + * Chat completions are called from browser-side clients as well as servers, so + * this route's streams carry the CORS header the other protocols do not need. + */ +export const CORS_HEADERS: Record = { + 'Access-Control-Allow-Origin': '*', +}; + +export interface OpenAIMessage { + role?: string; + content?: unknown; + tool_calls?: unknown[]; + tool_call_id?: string; + /** + * Prior-turn reasoning for an assistant message. + * + * Not an OpenAI field. The CodeBuddy chat upstream accepts it on assistant + * messages and uses it to carry reasoning across turns — the same slot + * CodeBuddy's own client populates when it replays a response. It is named + * `reasoning` rather than `reasoning_content` because that one is the + * upstream's *response* field; this is the request-side counterpart. + */ + reasoning?: string; +} + +export interface CacheableTextBlock { + cache_control?: { type: 'ephemeral' }; + text: string; + type: 'text'; +} + +export const MIN_AUTO_CACHE_TEXT_LENGTH = 1024; +export const MAX_STREAM_FRAME_LENGTH = 1_000_000; +export const CODEBUDDY_CLI_VERSION = '2.137.1'; +export const CODEBUDDY_USER_AGENT = `CLI/${CODEBUDDY_CLI_VERSION} CodeBuddy/${CODEBUDDY_CLI_VERSION}`; + +export interface ChatRequestBody { + model?: string; + messages?: OpenAIMessage[]; + stream?: boolean; + stream_options?: { + include_usage?: boolean; + }; + temperature?: number; + max_tokens?: number; + max_completion_tokens?: number; + response_format?: unknown; + top_p?: number; + frequency_penalty?: number; + presence_penalty?: number; + stop?: string | string[]; + tools?: unknown[]; + tool_choice?: unknown; + parallel_tool_calls?: boolean; + thinking?: Record; + reasoning_effort?: string; +} + +export interface ChatStreamDelta { + content?: string; + role?: string; + reasoning_content?: string; + reasoning?: string; + tool_calls?: Array<{ + index?: number; + id?: string; + type?: string; + function?: { + arguments?: string; + name?: string; + }; + }>; +} + +export interface ChatStreamChunk { + id?: string; + object?: string; + created?: number; + model?: string; + usage?: unknown; + choices?: Array<{ + delta?: ChatStreamDelta; + finish_reason?: string | null; + index?: number; + }>; +} + +export type ToolCallChunk = NonNullable[number]; + +export interface ToolCallMapping { + id: string; + index: number; +} + +export interface ToolCallNormalizationState { + mappings: Map; + nextIndex: number; +} + +export interface ResolvedAuth { + type: 'bearer'; + bearerToken: string; + userId: string; + credentialData: Record; +} + +export interface ProxyContext { + accessKeyId: string | null; + accessKeyName: string | null; + auth: ResolvedAuth; + credentialFilename: string | null; + preferences: { + firstMessageRoleToSystem: boolean; + firstSystemMessageRoleToUser: boolean; + upstreamProtocol: 'chat' | 'responses'; + }; +} + +export interface DiscoveredModel { + displayName: string; + id: string; +} + +export const SERVER_WEB_TOOL_NAMES = [ + normalizeToolName(WEB_SEARCH_TOOL_NAME), + normalizeToolName(WEB_FETCH_TOOL_NAME), +]; + +export interface StreamProbeState { + toolNames: Map; +} diff --git a/lib/server/proxy/codebuddy/upstream.ts b/lib/server/proxy/codebuddy/upstream.ts new file mode 100644 index 0000000..15d7e26 --- /dev/null +++ b/lib/server/proxy/codebuddy/upstream.ts @@ -0,0 +1,272 @@ +import type { NextRequest } from 'next/server'; + +import { getCodeBuddyApiEndpoint, getDefaultModel } from '../../domain/config'; +import { getCredentialSupportedModels } from '../../domain/credentials'; +import { resolveHyChatThinking } from '../../shared/hy-thought-depth'; +import { getRequestHeaderMap } from '../../shared/http'; +import { getCredentialValue } from './context'; +import { + type CacheableTextBlock, + type ChatRequestBody, + CODEBUDDY_CLI_VERSION, + CODEBUDDY_USER_AGENT, + MIN_AUTO_CACHE_TEXT_LENGTH, + type OpenAIMessage, + type ProxyContext, + type ResolvedAuth, +} from './types'; + +export const hasPromptCacheControl = (content: unknown): boolean => { + return ( + Array.isArray(content) && + content.some( + (part) => !!part && typeof part === 'object' && 'cache_control' in part, + ) + ); +}; + +export const createCacheableTextBlock = (text: string): CacheableTextBlock => ({ + type: 'text', + text, + cache_control: { type: 'ephemeral' }, +}); + +export const addPromptCacheControl = ( + message: OpenAIMessage, +): OpenAIMessage => { + if ( + typeof message.content === 'string' && + message.content.trim().length >= MIN_AUTO_CACHE_TEXT_LENGTH + ) { + return { + ...message, + content: [createCacheableTextBlock(message.content)], + }; + } + + if (Array.isArray(message.content)) { + const textIndex = message.content.findIndex( + (part) => + !!part && + typeof part === 'object' && + (part as { type?: unknown }).type === 'text' && + typeof (part as { text?: unknown }).text === 'string' && + (part as { text: string }).text.trim().length >= + MIN_AUTO_CACHE_TEXT_LENGTH, + ); + + if (textIndex >= 0) { + return { + ...message, + content: message.content.map((part, index) => + index === textIndex && part && typeof part === 'object' + ? { + ...part, + cache_control: { type: 'ephemeral' }, + } + : part, + ), + }; + } + } + + return message; +}; + +export const applyPromptCacheControl = ( + messages: OpenAIMessage[], +): OpenAIMessage[] => { + const explicitCacheControl = messages.some((message) => + hasPromptCacheControl(message.content), + ); + + if (explicitCacheControl) { + return messages; + } + + const cacheableIndexes = new Set(); + const systemIndex = messages.findIndex( + (message) => message.role === 'system', + ); + + if (systemIndex >= 0) { + cacheableIndexes.add(systemIndex); + } + + let lastUserIndex = -1; + + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index]?.role === 'user') { + lastUserIndex = index; + break; + } + } + + if (lastUserIndex >= 0) { + cacheableIndexes.add(lastUserIndex); + } + + if (cacheableIndexes.size === 0) { + return messages; + } + + return messages.map((message, index) => + cacheableIndexes.has(index) ? addPromptCacheControl(message) : message, + ); +}; + +export const normalizeMessages = ( + messages: OpenAIMessage[], + firstMessageRoleToSystem: boolean, + firstSystemMessageRoleToUser: boolean, +): OpenAIMessage[] => { + const filtered = messages.filter( + (item) => item.role && item.content !== undefined, + ); + + const firstSystemIndex = firstSystemMessageRoleToUser + ? filtered.findIndex((message) => message.role === 'system') + : -1; + const normalized = filtered.map((message, index) => { + if ( + (firstMessageRoleToSystem && message.role === 'developer') || + index === firstSystemIndex + ) { + return { ...message, role: 'user' }; + } + + return message; + }); + + // Preserve role:'tool' messages so the OpenAI-compatible upstream + // receives a valid tool_calls/tool-result pair for multi-step tool loops. + return applyPromptCacheControl(normalized); +}; + +export const buildUpstreamHeaders = async ( + request: NextRequest, + auth: ResolvedAuth, +): Promise => { + const baseUrl = new URL(await getCodeBuddyApiEndpoint()); + const incoming = getRequestHeaderMap(request.headers); + const requestId = + incoming['x-request-id'] ?? crypto.randomUUID().replaceAll('-', ''); + const conversationId = incoming['x-conversation-id'] ?? crypto.randomUUID(); + const conversationRequestId = + incoming['x-conversation-request-id'] ?? + crypto.randomUUID().replaceAll('-', ''); + const conversationMessageId = + incoming['x-conversation-message-id'] ?? + crypto.randomUUID().replaceAll('-', ''); + const headers = new Headers(incoming); + headers.set('Accept', 'application/json'); + headers.set('Authorization', `Bearer ${auth.bearerToken}`); + headers.set('Content-Type', 'application/json'); + headers.set('Host', baseUrl.host); + headers.set('User-Agent', CODEBUDDY_USER_AGENT); + headers.set('X-Agent-Intent', 'craft'); + headers.set('X-Conversation-ID', conversationId); + headers.set('X-Conversation-Message-ID', conversationMessageId); + headers.set('X-Conversation-Request-ID', conversationRequestId); + headers.set('X-IDE-Name', 'CLI'); + headers.set('X-IDE-Type', 'CLI'); + headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); + headers.set('X-Client-Platform', 'web'); + headers.set('X-Product', 'SaaS'); + headers.set('X-Product-Version', CODEBUDDY_CLI_VERSION); + headers.set('X-Request-ID', requestId); + headers.set('X-Requested-With', 'XMLHttpRequest'); + headers.set('X-User-Id', auth.userId); + headers.set('x-stainless-arch', process.arch); + headers.set('x-stainless-lang', 'js'); + headers.set('x-stainless-os', process.platform); + headers.set('x-stainless-package-version', CODEBUDDY_CLI_VERSION); + headers.set('x-stainless-retry-count', '0'); + headers.set('x-stainless-runtime', 'node'); + headers.set('x-stainless-runtime-version', process.version); + + const domain = getCredentialValue(auth.credentialData, ['domain']); + const enterpriseId = getCredentialValue(auth.credentialData, [ + 'enterprise_id', + 'enterpriseId', + ]); + const tenantId = + getCredentialValue(auth.credentialData, ['tenant_id', 'tenantId']) ?? + enterpriseId; + + if (domain) { + headers.set('X-Domain', String(domain)); + } + + if (enterpriseId) { + headers.set('X-Enterprise-Id', String(enterpriseId)); + } + + if (tenantId) { + headers.set('X-Tenant-Id', String(tenantId)); + } + + const origin = String(domain ?? '') + .toLowerCase() + .endsWith('workbuddy.ai') + ? 'https://www.workbuddy.ai' + : 'https://www.codebuddy.cn'; + headers.set('Content-Type', 'application/json'); + headers.set('Origin', origin); + headers.set('Referer', `${origin}/`); + headers.set('User-Agent', CODEBUDDY_USER_AGENT); + headers.set('X-Product', 'SaaS'); + headers.set('X-Requested-With', 'XMLHttpRequest'); + headers.set('X-IDE-Name', 'CLI'); + headers.set('X-IDE-Type', 'CLI'); + headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); + + return headers; +}; + +export const headersToRecord = ( + headers: HeadersInit, +): Record => { + return Object.fromEntries(new Headers(headers).entries()); +}; + +export const buildUpstreamBody = async ( + body: ChatRequestBody, + context: ProxyContext, +): Promise => { + const normalizedMessages = normalizeMessages( + body.messages ?? [], + context.preferences.firstMessageRoleToSystem, + context.preferences.firstSystemMessageRoleToUser, + ); + const maxTokens = body.max_tokens ?? body.max_completion_tokens; + const credentialModels = getCredentialSupportedModels( + context.auth.credentialData, + ); + const model = + typeof body.model === 'string' && body.model.trim() + ? body.model + : (credentialModels[0] ?? (await getDefaultModel())); + + const hyThinking = await resolveHyChatThinking(model, body); + + return { + model, + messages: normalizedMessages, + stream: true, + temperature: body.temperature, + max_tokens: maxTokens, + max_completion_tokens: body.max_completion_tokens ?? maxTokens, + response_format: body.response_format, + top_p: body.top_p, + frequency_penalty: body.frequency_penalty, + presence_penalty: body.presence_penalty, + stop: body.stop, + stream_options: body.stream_options, + tools: body.tools, + tool_choice: body.tool_choice, + parallel_tool_calls: body.parallel_tool_calls, + thinking: hyThinking.thinking, + reasoning_effort: hyThinking.reasoningEffort, + }; +}; diff --git a/lib/server/proxy/codebuddy/usage-stream.ts b/lib/server/proxy/codebuddy/usage-stream.ts new file mode 100644 index 0000000..05ae238 --- /dev/null +++ b/lib/server/proxy/codebuddy/usage-stream.ts @@ -0,0 +1,192 @@ +import { + createStreamCloser, + responsesStreamErrorChunks, + toUpstreamTimeoutMessage, +} from '../../shared/upstream-timeout'; +import { + extractResponsesId, + extractResponsesUsage, + recordProxyUsage, +} from './usage'; +import { MAX_STREAM_FRAME_LENGTH, type ProxyContext } from './types'; + +export const trackResponsesUsageStream = async ({ + fallbackUsage, + model, + onResponseId, + proxyContext, + upstreamResponse, +}: { + fallbackUsage: unknown; + model: string; + onResponseId?: (responseId: string) => Promise; + proxyContext: ProxyContext; + upstreamResponse: Response; +}): Promise => { + if (!upstreamResponse.body) { + await recordProxyUsage({ + model, + proxyContext, + route: '/v1/responses', + usage: fallbackUsage, + }); + + return new Response(null, { + headers: upstreamResponse.headers, + status: upstreamResponse.status, + }); + } + + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + const closer = createStreamCloser(); + let latestUsage = fallbackUsage; + let responseBinding: Promise | null = null; + let usageRecorded = false; + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + const recordStreamUsage = async (): Promise => { + if (usageRecorded) return; + usageRecorded = true; + try { + await recordProxyUsage({ + model, + proxyContext, + route: '/v1/responses', + usage: latestUsage, + }); + } catch (error) { + console.error('[CodeBuddy2API] Failed to record Responses stream usage', { + error, + route: '/v1/responses', + }); + } + }; + const bindResponseId = (id: string): Promise => { + if (!onResponseId) return Promise.resolve(); + responseBinding ??= onResponseId(id).catch((error) => { + console.error( + '[CodeBuddy2API] Failed to bind upstream Responses session', + { + error, + responseId: id, + }, + ); + }); + return responseBinding; + }; + const stream = new ReadableStream({ + start: (controller) => { + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + let responseId: string | null = null; + + const inspectFrame = async (frame: string): Promise => { + for (const line of frame.split('\n')) { + if (!line.startsWith('data:')) { + continue; + } + + const raw = line.slice(5).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + try { + const event = JSON.parse(raw) as unknown; + latestUsage = extractResponsesUsage(event) ?? latestUsage; + responseId = extractResponsesId(event) ?? responseId; + if (responseId) await bindResponseId(responseId); + } catch { + // Preserve malformed upstream frames without recording them. + } + } + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + if (buffer) { + await inspectFrame(buffer); + if (closer.closed) return; + controller.enqueue(encoder.encode(buffer)); + } + + await recordStreamUsage(); + await responseBinding; + releaseReader(); + closer.mark(); + controller.close(); + return; + } + + const text = decoder.decode(value, { stream: true }); + buffer += text; + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + buffer = ''; + } + + for (const frame of frames) { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + continue; + } + await inspectFrame(frame); + if (cancelled) return; + controller.enqueue(encoder.encode(`${frame}\n\n`)); + } + } + }; + + void pump().catch(async (error) => { + if (cancelled) return; + console.error('[CodeBuddy2API] Responses upstream stream failed', { + error, + route: '/v1/responses', + }); + await responseBinding; + await recordStreamUsage(); + releaseReader(); + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage !== null) { + // Frames here are forwarded verbatim, so the error has to arrive as + // the Responses protocol's own event. + closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); + return; + } + + controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + await responseBinding; + await recordStreamUsage(); + releaseReader(); + } + }, + }); + + return new Response(stream, { + headers: upstreamResponse.headers, + status: upstreamResponse.status, + }); +}; diff --git a/lib/server/proxy/codebuddy/usage.ts b/lib/server/proxy/codebuddy/usage.ts new file mode 100644 index 0000000..f814eee --- /dev/null +++ b/lib/server/proxy/codebuddy/usage.ts @@ -0,0 +1,152 @@ +import { recordUsageEvent, type UsageSnapshot } from '../../domain/usage'; +import type { ProxyContext } from './types'; + +export const toUsageSnapshot = (usage: unknown): UsageSnapshot | null => { + if (!usage || typeof usage !== 'object') { + return null; + } + + return usage as UsageSnapshot; +}; + +export const recordProxyUsage = async ({ + model, + proxyContext, + route, + usage, +}: { + model: string; + proxyContext: ProxyContext; + route: string; + usage: unknown; +}): Promise => { + await recordUsageEvent({ + accessKeyId: proxyContext.accessKeyId, + accessKeyName: proxyContext.accessKeyName, + credentialFilename: proxyContext.credentialFilename, + model, + route, + usage: toUsageSnapshot(usage) ?? {}, + }); +}; + +export const extractResponsesUsage = (value: unknown): unknown => { + if (!value || typeof value !== 'object') { + return null; + } + + const payload = value as { + response?: { + usage?: unknown; + }; + usage?: unknown; + }; + + return payload.response?.usage ?? payload.usage ?? null; +}; + +export const mapResponsesUsageToChat = ( + usage: unknown, +): Record | null => { + if (!usage || typeof usage !== 'object') return null; + + const value = usage as { + cache_creation_input_tokens?: unknown; + cache_read_input_tokens?: unknown; + input_tokens?: unknown; + input_tokens_details?: { + cache_creation_tokens?: unknown; + cached_tokens?: unknown; + }; + output_tokens?: unknown; + output_tokens_details?: { + reasoning_tokens?: unknown; + }; + total_tokens?: unknown; + }; + const inputTokens = Number(value.input_tokens ?? 0); + const outputTokens = Number(value.output_tokens ?? 0); + const cachedTokens = Number( + value.input_tokens_details?.cached_tokens ?? + value.cache_read_input_tokens ?? + 0, + ); + const cacheCreationTokens = Number( + value.input_tokens_details?.cache_creation_tokens ?? + value.cache_creation_input_tokens ?? + 0, + ); + const reasoningTokens = Number( + value.output_tokens_details?.reasoning_tokens ?? 0, + ); + + return { + completion_tokens: outputTokens, + completion_tokens_details: { + reasoning_tokens: reasoningTokens, + }, + prompt_tokens: inputTokens, + prompt_tokens_details: { + cache_creation_tokens: cacheCreationTokens, + cached_tokens: cachedTokens, + }, + total_tokens: Number(value.total_tokens ?? inputTokens + outputTokens), + }; +}; + +export const extractResponsesId = (value: unknown): string | null => { + if (!value || typeof value !== 'object') return null; + const payload = value as { + id?: unknown; + response?: { id?: unknown }; + }; + const id = payload.response?.id ?? payload.id; + return typeof id === 'string' && id ? id : null; +}; + +export const parseUsageHeader = (response: Response): unknown => { + const usageHeader = response.headers.get('x-codebuddy-usage'); + + if (!usageHeader) { + return null; + } + + try { + return JSON.parse(usageHeader) as unknown; + } catch { + return null; + } +}; + +export const logUpstreamFailure = ({ + detail, + error, + route, + status, + url, +}: { + detail?: string; + error?: unknown; + route: string; + status?: number; + url: string; +}): void => { + const payload: Record = { + route, + url, + }; + + if (typeof status === 'number') { + payload.status = status; + } + + if (detail) { + payload.detail = detail.slice(0, 1000); + } + + if (error) { + payload.error = error; + } + + console.error('[CodeBuddy2API] Upstream request failed', payload); +}; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 5f7f046..0bd4641 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -1,2760 +1,47 @@ -import type { NextRequest } from 'next/server'; - -import { - getDefaultModel, - isWebFetchEnabled, - isWebSearchEnabled, -} from '../domain/config'; -import { getCredentialSupportedModels } from '../domain/credentials'; -import type { DebugTrace } from '../domain/debug'; -import { - buildWebFetchToolDefinition, - buildWebSearchToolDefinition, - markServerTool, - normalizeToolName, - WEB_FETCH_TOOL_NAME, - WEB_FETCH_TOOL_TYPE_PREFIX, - WEB_SEARCH_TOOL_NAME, - WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; -import { - buildImageGenerationChatTool, - buildResponsesImageGenerationCallItem, - executeImageGenerationLoop, - IMAGE_GENERATION_CHAT_TOOL_NAME, - IMAGE_GENERATION_TOOL_TYPE, - type ImageGenerationExecution, -} from './image-generation'; -import { - extractImageUrl, - isImageContentPart, - proxyChatCompletions, - proxyResponsesUpstream, - resolveProxyContext, - resolveProxyContextByCredentialFilename, - type ProxyContext, -} from './codebuddy'; -import { - getServerToolExecutions, - type ServerToolExecution, - type ServerToolInvocation, -} from './web-search-loop'; -import { resolveRequestAccessKey } from './auth'; -import { createErrorResponse } from '../shared/http'; -import { - createStreamCloser, - readTimeoutFrame, - responsesStreamErrorChunks, - toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { - deleteStorageJson, - getStorageBackendMeta, - listStorageJson, - readStorageJson, - writeStorageJson, -} from '../storage'; - -interface ResponsesInputItem { - type?: string; - role?: string; - content?: unknown; - text?: string; - arguments?: string; - output?: unknown; - name?: string; - call_id?: string; - tools?: Array<{ type?: string; name?: string } & Record>; - /** - * Present on `reasoning` items a client replays from an earlier response. - * We put the reasoning here verbatim; clients echo it back untouched. - * A compaction item carries the same field. - */ - encrypted_content?: string; - /** - * Reasoning summaries. The Agents SDK sends these back as - * `summary: [{type: 'summary_text', text}]`. - */ - summary?: unknown; -} - -interface SupportedChatTool { - chatName: string; - kind: 'custom' | 'function' | 'mcp' | 'tool_search'; - namespace?: string; - originalName: string; - /** - * True when the client declared this as a provider-executed server tool - * (`web_search_20260209`, `web_fetch_20250910`, `web_search_preview`) rather - * than as its own function. - * - * Translation turns both into ordinary functions for upstream, so without this - * the proxy cannot tell them apart later — and the difference decides whether a - * tool that cannot be executed is dropped or forwarded. - */ - serverDeclared?: boolean; - serverLabel?: string; - tool: Record; -} - -interface ResponsesRequestBody { - model?: string; - input?: string | ResponsesInputItem[]; - instructions?: string; - messages?: Array<{ role?: string; content?: unknown }>; - stream?: boolean; - metadata?: Record; - reasoning?: Record; - thinking?: Record; - tools?: Array<{ type?: string; name?: string } & Record>; - tool_choice?: unknown; - max_output_tokens?: number; - previous_response_id?: string; -} - -type ResponseSessionDefaults = Pick< - ResponsesRequestBody, - 'instructions' | 'metadata' | 'tools' | 'tool_choice' ->; - -interface ResponseSession { - accessKeyId: string | null; - credentialFilename: string | null; - createdAt: number; - id: string; - model: string; - transcript: TranscriptMessage[]; - defaults: ResponseSessionDefaults; - upstreamProtocol?: 'chat' | 'responses'; -} - -interface ChatResponseToolCall { - index?: number; - id?: string; - type?: string; - function?: { - arguments?: string; - name?: string; - }; -} - -interface ChatResponseMessage { - content?: unknown; - tool_calls?: ChatResponseToolCall[]; - /** Reasoning the upstream produced alongside `content`. */ - reasoning_content?: string; - reasoning?: string; -} - -interface ChatImagePart { - image_url: { url: string }; - type: 'image_url'; -} - -interface ChatTextPart { - text: string; - type: 'text'; -} - -type ChatContentPart = string | ChatTextPart | ChatImagePart; - -/** - * Transcript content. Images are kept as structured parts so they survive the - * Chat-shaped round trip through the transcript and reach the model as images - * instead of a JSON dump. - */ -type TranscriptContent = string | ChatContentPart[]; - -interface TranscriptMessage { - role: string; - content: TranscriptContent | null; - tool_calls?: Array<{ - id: string; - type: string; - function: { - name: string; - arguments: string; - }; - }>; - tool_call_id?: string; - /** - * Prior-turn reasoning recovered from a replayed reasoning item. - * - * Carried on the assistant message the reasoning belongs to rather than sent - * as its own message: the chat upstream has no standalone reasoning entry, - * and a reasoning-only message would be an empty turn. - */ - reasoning?: string; -} - -interface StreamingToolCallState { - addedEmitted: boolean; - arguments: string; - canonicalKey: string; - callId: string; - name: string; - outputIndex: number; - outputItemId: string; - pendingArgumentDeltas: string[]; -} - -interface StreamingMessageState { - outputIndex: number | null; - outputItemId: string; -} - -interface ResponsesServerToolItem { - completed: Record; - inProgress: Record; - outputIndex: number; -} - -interface ResponseSessionMetadata { - bytes: number; - createdAt: number; -} - -type SupportedResponsesTool = NonNullable< - ResponsesRequestBody['tools'] ->[number]; - -const TOOL_SEARCH_PROXY_NAME = 'tool_search'; -const CUSTOM_TOOL_INPUT_FIELD = 'input'; -const CUSTOM_TOOL_INPUT_DESCRIPTION = - 'Raw string input for the original custom tool.'; -const MAX_RESPONSE_SESSIONS = 1_000; -const RESPONSE_SESSION_TTL_MS = 60 * 60 * 1000; -const MAX_RESPONSE_SESSION_BYTES = 8 * 1024 * 1024; -const MAX_RESPONSE_SESSION_TOTAL_BYTES = 64 * 1024 * 1024; -const MAX_RESPONSE_TRANSCRIPT_MESSAGES = 200; -const RESPONSE_SESSION_NAMESPACE = 'responses'; -const RESPONSE_SESSION_INDEX_NAMESPACE = 'response-session-index'; -const MAX_STREAM_BUFFER_LENGTH = 1_000_000; -const MAX_STREAM_TEXT_LENGTH = 2_000_000; -const MAX_TOOL_ARGUMENT_LENGTH = 1_000_000; -const MAX_TOOL_NAME_LENGTH = 256; - -const globalResponsesState = globalThis as typeof globalThis & { - __codebuddy2apiResponseSessions__?: Map; - __codebuddy2apiResponseSessionBytes__?: Map; - __codebuddy2apiResponseSessionTotalBytes__?: number; -}; - -const getSessionStore = (): Map => { - if (!globalResponsesState.__codebuddy2apiResponseSessions__) { - globalResponsesState.__codebuddy2apiResponseSessions__ = new Map(); - } - - return globalResponsesState.__codebuddy2apiResponseSessions__; -}; - -const getSessionByteStore = (): Map => { - if (!globalResponsesState.__codebuddy2apiResponseSessionBytes__) { - globalResponsesState.__codebuddy2apiResponseSessionBytes__ = new Map(); - } - - return globalResponsesState.__codebuddy2apiResponseSessionBytes__; -}; - -const getSessionTotalBytes = (): number => { - return globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ ?? 0; -}; - -const setSessionTotalBytes = (value: number): void => { - globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ = value; -}; - -const removeLocalResponseSession = (id: string): void => { - const byteStore = getSessionByteStore(); - const store = getSessionStore(); - const bytes = byteStore.get(id) ?? 0; - store.delete(id); - byteStore.delete(id); - setSessionTotalBytes(Math.max(0, getSessionTotalBytes() - bytes)); -}; - -const pruneResponseSessions = (): void => { - const store = getSessionStore(); - const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; - - for (const [id, session] of store) { - if (session.createdAt <= expiresBefore) { - removeLocalResponseSession(id); - } - } - - while ( - store.size > MAX_RESPONSE_SESSIONS || - getSessionTotalBytes() > MAX_RESPONSE_SESSION_TOTAL_BYTES - ) { - const oldestId = store.keys().next().value; - - // Guard against a byte total that has drifted out of step with the map. - // Without this, an empty map with a positive total makes the removal a - // no-op and spins here forever, blocking the event loop. - if (oldestId === undefined) { - setSessionTotalBytes(0); - break; - } - - removeLocalResponseSession(oldestId); - } -}; - -const prunePgResponseSessions = async (): Promise => { - const metadataDocuments = await listStorageJson( - RESPONSE_SESSION_INDEX_NAMESPACE, - ); - const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; - const candidates = metadataDocuments - .map((document) => ({ key: document.key, ...document.value })) - .sort((left, right) => left.createdAt - right.createdAt); - const toDelete = candidates.filter( - (candidate) => candidate.createdAt <= expiresBefore, - ); - const remaining = candidates.filter( - (candidate) => candidate.createdAt > expiresBefore, - ); - let totalBytes = remaining.reduce( - (total, candidate) => total + candidate.bytes, - 0, - ); - - while ( - remaining.length > MAX_RESPONSE_SESSIONS || - totalBytes > MAX_RESPONSE_SESSION_TOTAL_BYTES - ) { - const candidate = remaining.shift()!; - toDelete.push(candidate); - totalBytes -= candidate.bytes; - } - - await Promise.all( - toDelete.flatMap((candidate) => [ - deleteStorageJson(RESPONSE_SESSION_NAMESPACE, candidate.key), - deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, candidate.key), - ]), - ); -}; - -const isPgResponseSessionStore = (): boolean => { - return getStorageBackendMeta().backend === 'pg'; -}; - -const getResponseSession = async ( - id: string, -): Promise => { - if (isPgResponseSessionStore()) { - const session = await readStorageJson( - RESPONSE_SESSION_NAMESPACE, - id, - ); - if (!session || session.createdAt <= Date.now() - RESPONSE_SESSION_TTL_MS) { - if (session) { - await deleteStorageJson(RESPONSE_SESSION_NAMESPACE, id); - await deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, id); - } - return undefined; - } - return session; - } - - pruneResponseSessions(); - return getSessionStore().get(id); -}; - -const getValidatedPreviousSession = async ( - previousResponseId: string | null, - accessKeyId: string | null, -): Promise => { - const previousSession = previousResponseId - ? await getResponseSession(previousResponseId) - : undefined; - - if ( - previousResponseId && - (!previousSession || previousSession.accessKeyId !== accessKeyId) - ) { - throw new Error('Unknown or expired previous_response_id'); - } - - return previousSession; -}; - -const storeResponseSession = async ( - session: ResponseSession, -): Promise => { - const serialized = JSON.stringify(session); - if (Buffer.byteLength(serialized, 'utf8') > MAX_RESPONSE_SESSION_BYTES) { - throw new Error('Response session exceeds the maximum size'); - } - - if (isPgResponseSessionStore()) { - await writeStorageJson(RESPONSE_SESSION_NAMESPACE, session.id, session); - await writeStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, session.id, { - bytes: Buffer.byteLength(serialized, 'utf8'), - createdAt: session.createdAt, - }); - try { - await prunePgResponseSessions(); - } catch (error) { - console.warn('[CodeBuddy2API] Unable to prune Responses sessions', error); - } - return; - } - - const store = getSessionStore(); - const byteStore = getSessionByteStore(); - const previousBytes = byteStore.get(session.id) ?? 0; - const sessionBytes = Buffer.byteLength(serialized, 'utf8'); - store.set(session.id, session); - byteStore.set(session.id, sessionBytes); - setSessionTotalBytes(getSessionTotalBytes() - previousBytes + sessionBytes); - pruneResponseSessions(); -}; - -const storeUpstreamResponseBinding = async ({ - model, - proxyContext, - responseId, -}: { - model: string; - proxyContext: ProxyContext; - responseId: string; -}): Promise => { - await storeResponseSession({ - accessKeyId: proxyContext.accessKeyId, - credentialFilename: proxyContext.credentialFilename, - createdAt: Date.now(), - defaults: {}, - id: responseId, - model, - transcript: [], - upstreamProtocol: 'responses', - }); -}; - -const flattenNamespaceToolName = (namespace: string, name: string): string => { - return `${namespace}__${name}`; -}; - -const extractFunctionDefinition = ( - tool: Record, -): Record | null => { - const nested = - typeof tool.function === 'object' && tool.function !== null - ? (tool.function as Record) - : {}; - - const name = nested.name ?? tool.name; - if (typeof name !== 'string' || name.length === 0) { - return null; - } - - const functionDef: Record = { name }; - - const description = nested.description ?? tool.description; - if (description !== undefined) { - functionDef.description = description; - } - - const parameters = nested.parameters ?? tool.parameters; - if (parameters !== undefined) { - functionDef.parameters = parameters; - } - - const strict = nested.strict ?? tool.strict; - if (strict !== undefined) { - functionDef.strict = strict; - } - - return functionDef; -}; - -const buildCustomToolDefinition = ( - tool: Record, -): Record | null => { - const name = typeof tool.name === 'string' ? tool.name.trim() : ''; - - if (!name) { - return null; - } - - const description = - typeof tool.description === 'string' && tool.description.trim() - ? tool.description - : `Custom tool ${name}`; - - return { - name, - description, - parameters: { - type: 'object', - properties: { - [CUSTOM_TOOL_INPUT_FIELD]: { - type: 'string', - description: CUSTOM_TOOL_INPUT_DESCRIPTION, - }, - }, - required: [CUSTOM_TOOL_INPUT_FIELD], - }, - }; -}; - -const buildToolSearchDefinition = (): Record => { - return { - name: TOOL_SEARCH_PROXY_NAME, - description: - 'Search and load Codex tools, plugins, connectors, and MCP namespaces for the current task.', - parameters: { - type: 'object', - properties: { - query: { - type: 'string', - description: 'Search query for tools or connectors to load.', - }, - limit: { - type: 'integer', - description: 'Maximum number of tool groups to return.', - }, - }, - required: ['query'], - }, - }; -}; - -const toSupportedChatTool = ( - tool: SupportedResponsesTool, - namespace?: string, -): SupportedChatTool[] => { - const toolType = typeof tool.type === 'string' ? tool.type : 'function'; - - // Server-side search and fetch carry no function schema, so the generic - // branch below drops them. Emit them as functions unconditionally and let - // the proxy loop resolve the configured backend asynchronously. SearXNG - // needs local configuration, while CodeBuddy search does not. - if ( - normalizeToolName(toolType).startsWith( - normalizeToolName(WEB_SEARCH_TOOL_TYPE_PREFIX), - ) - ) { - const definition = buildWebSearchToolDefinition(); - - return [ - { - chatName: WEB_SEARCH_TOOL_NAME, - kind: 'function', - originalName: WEB_SEARCH_TOOL_NAME, - serverDeclared: true, - tool: definition, - }, - ]; - } - - // Fetch needs no deployment-level configuration — the local backend is always - // available and the CodeBuddy backend needs only a credential — so it is - // advertised unconditionally and gated later by the enable toggle. - if ( - normalizeToolName(toolType).startsWith( - normalizeToolName(WEB_FETCH_TOOL_TYPE_PREFIX), - ) - ) { - const definition = buildWebFetchToolDefinition(); - - return [ - { - chatName: WEB_FETCH_TOOL_NAME, - kind: 'function', - originalName: WEB_FETCH_TOOL_NAME, - serverDeclared: true, - tool: definition, - }, - ]; - } - - // Image generation has no chat-protocol equivalent, so the declaration is - // rewritten as a function and the call is executed by the proxy. It is - // advertised only on the chat path: the responses passthrough hands the - // native declaration straight to the upstream, which supports it. - if (toolType === IMAGE_GENERATION_TOOL_TYPE) { - return [ - { - chatName: IMAGE_GENERATION_CHAT_TOOL_NAME, - kind: 'function' as const, - originalName: IMAGE_GENERATION_TOOL_TYPE, - serverDeclared: true, - tool: buildImageGenerationChatTool(), - }, - ]; - } - - if (toolType === 'namespace') { - const namespaceName = typeof tool.name === 'string' ? tool.name.trim() : ''; - const children = ( - Array.isArray(tool.tools) - ? tool.tools - : Array.isArray(tool.children) - ? tool.children - : [] - ).filter((item): item is SupportedResponsesTool => { - return Boolean(item && typeof item === 'object'); - }); - - if (!namespaceName || !children.length) { - return []; - } - - return children.flatMap((child) => - toSupportedChatTool(child, namespaceName), - ); - } - - if (toolType === 'tool_search') { - const definition = buildToolSearchDefinition(); - return [ - { - chatName: TOOL_SEARCH_PROXY_NAME, - kind: 'tool_search', - originalName: TOOL_SEARCH_PROXY_NAME, - tool: definition, - }, - ]; - } - - if (toolType === 'custom') { - const definition = buildCustomToolDefinition(tool); - - if (!definition || typeof definition.name !== 'string') { - return []; - } - - return [ - { - chatName: definition.name, - kind: 'custom', - originalName: definition.name, - tool: definition, - }, - ]; - } - - const functionDef = extractFunctionDefinition(tool); - - if (!functionDef || typeof functionDef.name !== 'string') { - return []; - } - - const originalName = functionDef.name; - const chatName = namespace - ? flattenNamespaceToolName(namespace, originalName) - : toolType === 'mcp' && - typeof tool.server_label === 'string' && - tool.server_label.trim() - ? flattenNamespaceToolName(tool.server_label.trim(), originalName) - : originalName; - - return [ - { - chatName, - kind: toolType === 'mcp' ? 'mcp' : 'function', - namespace: - namespace || - (typeof tool.server_label === 'string' ? tool.server_label : undefined), - originalName, - serverLabel: - toolType === 'mcp' && typeof tool.server_label === 'string' - ? tool.server_label - : undefined, - tool: { - ...functionDef, - name: chatName, - }, - }, - ]; -}; - -/** - * True when the client declared an `image_generation` tool. Gating on this - * keeps the ordinary path free of an extra upstream round trip. - */ -const hasImageGenerationTool = ( - tools: ResponsesRequestBody['tools'], -): boolean => { - return Boolean( - tools?.some( - (tool) => - typeof tool?.type === 'string' && - tool.type.toLowerCase().replaceAll('-', '_') === - IMAGE_GENERATION_TOOL_TYPE, - ), - ); -}; - -const getSupportedChatTools = ( - tools: ResponsesRequestBody['tools'], -): SupportedChatTool[] => { - if (!tools?.length) { - return []; - } - - return tools.flatMap((tool) => toSupportedChatTool(tool)); -}; - -const findSupportedToolByName = ( - tools: ResponsesRequestBody['tools'], - name: string, -): SupportedChatTool | null => { - if (!tools?.length || !name) { - return null; - } - - return ( - getSupportedChatTools(tools).find( - (tool) => tool.chatName === name || tool.originalName === name, - ) ?? null - ); -}; - -const hasSupportedLongerToolNamePrefix = ( - tools: ResponsesRequestBody['tools'], - prefix: string, -): boolean => { - if (!tools?.length || !prefix) { - return false; - } - - return getSupportedChatTools(tools).some((tool) => { - const name = tool.chatName; - return ( - typeof name === 'string' && - name.length > prefix.length && - name.startsWith(prefix) - ); - }); -}; - -const buildResponsesToolCallOutputItem = ( - tools: ResponsesRequestBody['tools'], - toolCall: { - arguments: string; - callId: string; - id: string; - name: string; - status: 'completed' | 'in_progress'; - }, -): Record => { - const originalTool = findSupportedToolByName(tools, toolCall.name); - const itemType = originalTool?.kind === 'mcp' ? 'mcp_call' : 'function_call'; - const item: Record = { - id: toolCall.id, - type: itemType, - call_id: toolCall.callId, - name: originalTool?.originalName ?? toolCall.name ?? 'function', - arguments: toolCall.arguments, - status: toolCall.status, - }; - - if (originalTool?.kind === 'mcp' && originalTool.serverLabel) { - item.server_label = originalTool.serverLabel; - } - - if (originalTool?.kind === 'function' && originalTool.namespace) { - item.namespace = originalTool.namespace; - } - - return item; -}; - -const getResponsesToolCallArgumentDeltaEventType = ( - tools: ResponsesRequestBody['tools'], - name: string, -): - | 'response.function_call_arguments.delta' - | 'response.mcp_call_arguments.delta' => { - return findSupportedToolByName(tools, name)?.kind === 'mcp' - ? 'response.mcp_call_arguments.delta' - : 'response.function_call_arguments.delta'; -}; - -const buildAssistantTranscriptToolCalls = ( - toolCalls: ChatResponseToolCall[], - tools?: ResponsesRequestBody['tools'], -): TranscriptMessage['tool_calls'] | undefined => { - if (!toolCalls.length) { - return undefined; - } - - return toolCalls.map((toolCall, index) => ({ - id: normalizeToolCallId(toolCall.id, index), - type: 'function', - function: { - name: - findSupportedToolByName(tools, toolCall.function?.name ?? '') - ?.originalName ?? - toolCall.function?.name ?? - 'function', - arguments: toolCall.function?.arguments ?? '', - }, - })); -}; - -const buildStreamingAssistantTranscriptToolCalls = ( - toolCallStates: StreamingToolCallState[], - tools?: ResponsesRequestBody['tools'], -): TranscriptMessage['tool_calls'] | undefined => { - if (!toolCallStates.length) { - return undefined; - } - - return toolCallStates.map((toolCallState) => ({ - id: toolCallState.callId, - type: 'function', - function: { - arguments: toolCallState.arguments, - name: - findSupportedToolByName(tools, toolCallState.name)?.originalName ?? - toolCallState.name, - }, - })); -}; - -const getAssistantTranscriptContent = ( - outputText: string, - toolCalls: TranscriptMessage['tool_calls'] | undefined, -): string | null => { - return toolCalls?.length ? outputText || null : outputText; -}; - -/** - * Keeps image parts as structured content so the chat path can rebuild them - * upstream. Text parts are still flattened: the transcript is persisted across - * turns and replayed as Chat messages, and the Responses converter only - * recognises images in the OpenAI `image_url` shape. - */ -const mapInputContentToTranscriptContent = ( - content: unknown, -): TranscriptContent | null => { - if (typeof content === 'string') { - return content; - } - - if (!Array.isArray(content)) { - return null; - } - - const parts = content.filter((part) => part !== null && part !== undefined); - - if (!parts.some(isImageContentPart)) { - return null; - } - - const mapped = parts.flatMap((part): ChatContentPart[] => { - if (typeof part === 'string') { - return [part]; - } - - if (isImageContentPart(part)) { - const imageUrl = extractImageUrl(part); - - return imageUrl - ? [{ image_url: { url: imageUrl }, type: 'image_url' }] - : []; - } - - if (part && typeof part === 'object' && 'text' in part) { - return [String((part as { text?: unknown }).text ?? '')]; - } - - return []; - }); - - return mapped.length ? mapped : null; -}; - -const stringifyContent = (value: unknown): string => { - if (typeof value === 'string') { - return value; - } - - if (Array.isArray(value)) { - return value - .map((item) => { - if (typeof item === 'string') { - return item; - } - - if (item && typeof item === 'object' && 'text' in item) { - return String((item as { text?: unknown }).text ?? ''); - } - - return JSON.stringify(item); - }) - .join(''); - } - - if (value === undefined || value === null) { - return ''; - } - - return JSON.stringify(value); -}; - -/** - * Marks an `encrypted_content` value we minted, so we can tell it apart from a - * blob issued by someone else. - * - * Not a security measure. Codex never opens this field — it only echoes it — so - * plaintext round-trips fine, but a marker is what stops us from reading a - * genuinely encrypted blob as if it were reasoning text. - */ -const REASONING_PREFIX = 'cbreason1:'; - -/** - * Pulls readable reasoning out of a replayed `reasoning` item. - * - * Only values we minted are used: anything else — an OpenAI-issued blob, say — - * is opaque ciphertext, and forwarding it upstream would send gibberish where - * reasoning belongs. The summary is the fallback in that case. - * - * The Agents SDK sends summaries as `summary: [{type: 'summary_text', text}]`, - * so a client that never received our blob still gets its reasoning through. - */ -const extractReasoningFromItem = (item: ResponsesInputItem): string => { - const blob = item.encrypted_content; - - if (typeof blob === 'string' && blob.startsWith(REASONING_PREFIX)) { - return blob.slice(REASONING_PREFIX.length); - } - - if (!Array.isArray(item.summary)) { - return ''; - } - - return item.summary - .map((entry) => { - if (typeof entry === 'string') { - return entry; - } - - if (entry && typeof entry === 'object' && 'text' in entry) { - return String((entry as { text?: unknown }).text ?? ''); - } - - return ''; - }) - .join(''); -}; - -const mapInputItemToMessage = ( - item: ResponsesInputItem, -): TranscriptMessage | null => { - if (item.type === 'reasoning' || item.type === 'compaction') { - // Reasoning is not a message. Without this branch the item fell through to - // the plain-message case at the bottom, where it has neither `role` nor - // `content` — becoming an empty `{role:'user', content:''}` entry that the - // chat upstream sees as a turn the user never sent, repeated on every - // later turn of the conversation. - // - // Signal the reasoning back to the caller instead, which attaches it to the - // assistant message it accompanies. Returning `null` when there is nothing - // to recover keeps an empty reasoning item from emitting a message at all. - const reasoning = extractReasoningFromItem(item); - - return reasoning ? { role: 'assistant', content: null, reasoning } : null; - } - - if (item.type === 'function_call' || item.type === 'mcp_call') { - return { - role: 'assistant', - content: null, - tool_calls: [ - { - id: item.call_id ?? createResponseOutputId(), - type: 'function', - function: { - name: item.name ?? 'function', - arguments: item.arguments ?? '', - }, - }, - ], - }; - } - - if (item.type === 'function_call_output' || item.type === 'mcp_call_output') { - // A tool may return an image, e.g. a screenshot. Keep it structured so the - // Responses converter can rebuild it as an image; stringifying would hand - // the model the base64 payload as text. - const outputContent = - mapInputContentToTranscriptContent(item.output) ?? - stringifyContent(item.output); - - if (item.call_id) { - return { - role: 'tool', - content: outputContent, - tool_call_id: item.call_id, - }; - } - - return { - role: 'user', - content: outputContent, - }; - } - - if (item.type === 'mcp_approval_response') { - return { - role: 'user', - content: JSON.stringify(item), - }; - } - - // Every other item type returns above, so what is left is a plain message: - // either one with a declared `type: 'message'`, or one carrying only - // `role`/`content`. Images are kept structured so the chat path can rebuild - // them; a message without an image stays flattened. - const imageContent = mapInputContentToTranscriptContent(item.content); - - if (imageContent !== null) { - return { - role: item.role ?? 'user', - content: imageContent, - }; - } - - return { - role: item.role ?? 'user', - content: item.text ?? stringifyContent(item.content), - }; -}; - -const createResponseId = (): string => { - return `resp_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const createMessageId = (): string => { - return `msg_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const createResponseReasoningId = (): string => { - return `rs_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const createResponseOutputId = (): string => { - return `fc_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const normalizeToolCallId = (id: string | undefined, index: number): string => { - if (id && !id.startsWith('tooluse_')) { - return id; - } - - return `call_${id?.replace(/^tooluse_/, '') ?? index + 1}`; -}; - -export const translateResponsesToolsToChat = ( - tools: ResponsesRequestBody['tools'], -): unknown[] | undefined => { - if (!tools?.length) { - return undefined; - } - - const supported = getSupportedChatTools(tools); - if (!supported.length) { - return undefined; - } - - return supported.map((tool) => { - return { - type: 'function', - function: tool.tool, - ...(tool.serverDeclared ? markServerTool({}) : {}), - }; - }); -}; - -const translateResponsesToolChoiceToChat = (toolChoice: unknown): unknown => { - if (typeof toolChoice !== 'object' || toolChoice === null) { - return toolChoice; - } - - const choice = toolChoice as Record; - - if ( - choice.type === 'function' && - choice.function && - typeof choice.function === 'object' - ) { - return toolChoice; - } - - if ( - (choice.type === 'auto' || - choice.type === 'none' || - choice.type === 'required') && - typeof choice.type === 'string' - ) { - return choice.type; - } - - // Responses API selects a function by name: - // {type: 'function', name: 'fn'} -> chat schema {type: 'function', function: {name: 'fn'}} - if (typeof choice.name === 'string') { - return { - type: 'function', - function: { name: choice.name }, - }; - } - - return toolChoice; -}; - -const translateResponsesToolChoiceToChatWithTools = ( - tools: ResponsesRequestBody['tools'], - toolChoice: unknown, -): unknown => { - const translated = translateResponsesToolChoiceToChat(toolChoice); - - if (typeof translated !== 'object' || translated === null) { - return translated; - } - - const choice = translated as Record; - - if ( - choice.type === 'function' && - typeof choice.function === 'object' && - choice.function !== null - ) { - const functionChoice = choice.function as Record; - if (typeof functionChoice.name === 'string') { - return { - ...choice, - function: { - ...functionChoice, - name: resolveChatToolName(tools, functionChoice.name), - }, - }; - } - } - - return translated; -}; - -const getNamedToolChoice = (toolChoice: unknown): string | null => { - if (typeof toolChoice !== 'object' || toolChoice === null) { - return null; - } - - const choice = toolChoice as Record; - - if (typeof choice.name === 'string' && choice.name.length > 0) { - return choice.name; - } - - if ( - choice.type === 'function' && - typeof choice.function === 'object' && - choice.function !== null && - typeof (choice.function as Record).name === 'string' - ) { - return (choice.function as Record).name; - } - - return null; -}; - -const getResponsesCompatibilityError = ( - tools: ResponsesRequestBody['tools'], - toolChoice: unknown, -): Response | null => { - const supportedTools = getSupportedChatTools(tools); - - if (toolChoice === 'required' && supportedTools.length === 0) { - return createErrorResponse( - 400, - 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', - ); - } - - if (typeof toolChoice === 'object' && toolChoice !== null) { - const choice = toolChoice as Record; - const isPretranslatedFunctionChoice = - choice.type === 'function' && - typeof choice.function === 'object' && - choice.function !== null; - const isSimpleChoiceType = - choice.type === 'auto' || - choice.type === 'none' || - choice.type === 'required'; - const isNamedFunctionLikeChoice = typeof choice.name === 'string'; - - if ( - !isPretranslatedFunctionChoice && - !isSimpleChoiceType && - !isNamedFunctionLikeChoice - ) { - return createErrorResponse( - 400, - 'Unsupported Responses tool_choice for this /v1/responses adapter', - ); - } - - if (choice.type === 'required' && supportedTools.length === 0) { - return createErrorResponse( - 400, - 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', - ); - } - } - - const namedToolChoice = getNamedToolChoice(toolChoice); - if (namedToolChoice) { - const supportedNames = new Set( - supportedTools - .map((tool) => tool.originalName) - .filter((name): name is string => typeof name === 'string'), - ); - - if (!supportedNames.has(namedToolChoice)) { - return createErrorResponse( - 400, - 'tool_choice references a tool that is not available to this /v1/responses adapter', - ); - } - } - - return null; -}; - -const getStreamingToolCallCanonicalKey = ( - toolCall: ChatResponseToolCall, - position: number, -): string => { - if (toolCall.id) { - return `id:${toolCall.id}`; - } - - if (typeof toolCall.index === 'number') { - return `index:${toolCall.index}`; - } - - return `position:${position}`; -}; - -const getStreamingToolCallLookupKeys = ( - toolCall: ChatResponseToolCall, - position: number, -): string[] => { - if (toolCall.id || typeof toolCall.index === 'number') { - return [ - toolCall.id ? `id:${toolCall.id}` : null, - typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, - ].filter((key): key is string => key !== null); - } - - return [`position:${position}`]; -}; - -const prepareTranscript = async ( - body: ResponsesRequestBody, - accessKeyId: string | null, - previousSession?: ResponseSession, -): Promise<{ - defaults: ResponseSessionDefaults; - model: string; - transcript: TranscriptMessage[]; - previousResponseId: string | null; -}> => { - const previousResponseId = body.previous_response_id ?? null; - const resolvedPreviousSession = - previousSession ?? - (await getValidatedPreviousSession(previousResponseId, accessKeyId)); - - const transcript = (resolvedPreviousSession?.transcript ?? []).slice( - -MAX_RESPONSE_TRANSCRIPT_MESSAGES, - ); - // Reasoning recovered from replayed reasoning items, awaiting the assistant - // message it belongs to. Declared here so it spans the whole input array. - let pendingReasoning = ''; - while (transcript[0]?.role === 'tool') { - transcript.shift(); - } - const model = - typeof body.model === 'string' && body.model.trim() - ? body.model - : (resolvedPreviousSession?.model ?? (await getDefaultModel())); - const additionalTools = Array.isArray(body.input) - ? body.input.flatMap((item) => - item?.type === 'additional_tools' && Array.isArray(item.tools) - ? item.tools - : [], - ) - : []; - const baseTools = body.tools ?? resolvedPreviousSession?.defaults.tools; - const requestTools = [...(baseTools ?? []), ...additionalTools]; - const defaults = { - instructions: - body.instructions ?? - resolvedPreviousSession?.defaults.instructions ?? - undefined, - metadata: - body.metadata ?? resolvedPreviousSession?.defaults.metadata ?? undefined, - tools: requestTools.length > 0 ? requestTools : baseTools, - tool_choice: - body.tool_choice ?? - resolvedPreviousSession?.defaults.tool_choice ?? - undefined, - }; - - if (body.messages?.length) { - body.messages.forEach((item) => { - transcript.push({ - role: item.role ?? 'user', - content: - mapInputContentToTranscriptContent(item.content) ?? - stringifyContent(item.content), - }); - }); - } else if (typeof body.input === 'string') { - transcript.push({ role: 'user', content: body.input }); - } else if (Array.isArray(body.input)) { - body.input.forEach((item) => { - if (item.type === 'additional_tools') return; - - const message = mapInputItemToMessage(item); - - if (!message) { - return; - } - - // A reasoning item yields a reasoning-only entry. Fold it into the next - // assistant message so the upstream sees the reasoning where it belongs - // — attached to the turn that produced it — instead of as a bare turn. - // Anything left unconsumed at the end is dropped: reasoning with no - // following assistant message has nothing to attach to. - if (message.reasoning && !message.content && !message.tool_calls) { - pendingReasoning += message.reasoning; - return; - } - - // Attach any reasoning carried forward from a preceding reasoning item. - // `message.reasoning` is only ever set by the mapper below — clients - // cannot send it, since `ResponsesInputItem` has no such field — so - // there is no pre-existing value to merge with. - if (pendingReasoning) { - message.reasoning = pendingReasoning; - pendingReasoning = ''; - } - - transcript.push(message); - }); - } - - return { - defaults, - model, - transcript, - previousResponseId, - }; -}; - -const resolveChatToolName = ( - tools: ResponsesRequestBody['tools'], - name: string, -): string => { - return findSupportedToolByName(tools, name)?.chatName ?? name; -}; - -const normalizeTranscriptMessageToolNames = ( - transcript: TranscriptMessage[], - tools: ResponsesRequestBody['tools'], -): TranscriptMessage[] => { - return transcript.map((message) => { - if (!message.tool_calls?.length) { - return message; - } - - return { - ...message, - tool_calls: message.tool_calls.map((toolCall) => ({ - ...toolCall, - function: { - ...toolCall.function, - name: resolveChatToolName(tools, toolCall.function.name), - }, - })), - }; - }); -}; - -const toResponsesUsageNumber = (value: unknown): number => { - const numeric = - typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); - - if (!Number.isFinite(numeric) || numeric < 0) { - return 0; - } - - return numeric; -}; - -const mapChatUsageToResponses = (usage: unknown): Record => { - if (!usage || typeof usage !== 'object') { - return { - input_tokens: 0, - input_tokens_details: { cached_tokens: 0 }, - output_tokens: 0, - output_tokens_details: { reasoning_tokens: 0 }, - total_tokens: 0, - }; - } - - const value = usage as { - cache_creation_input_tokens?: unknown; - cache_read_input_tokens?: unknown; - completion_tokens?: unknown; - completion_tokens_details?: { reasoning_tokens?: unknown }; - completion_thinking_tokens?: unknown; - input_tokens_details?: { cached_tokens?: unknown }; - prompt_cache_hit_tokens?: unknown; - prompt_cache_miss_tokens?: unknown; - prompt_cache_write_tokens?: unknown; - prompt_tokens?: unknown; - prompt_tokens_details?: { - cache_creation_tokens?: unknown; - cached_tokens?: unknown; - }; - total_tokens?: unknown; - }; - const outputTokens = toResponsesUsageNumber(value.completion_tokens); - const cachedTokens = toResponsesUsageNumber( - value.prompt_tokens_details?.cached_tokens ?? - value.input_tokens_details?.cached_tokens ?? - value.cache_read_input_tokens ?? - value.prompt_cache_hit_tokens, - ); - const cacheCreationTokens = toResponsesUsageNumber( - value.prompt_tokens_details?.cache_creation_tokens ?? - value.cache_creation_input_tokens ?? - value.prompt_cache_write_tokens, - ); - const reasoningTokens = toResponsesUsageNumber( - value.completion_tokens_details?.reasoning_tokens ?? - value.completion_thinking_tokens, - ); - // Chat usage is the single source of truth for both shapes. Keep the - // Responses counters faithful to it so clients never see zeroed metrics. - // prompt_tokens already covers its cached and created subsets, so the - // split counters are only summed when prompt_tokens is missing. Otherwise - // cached tokens would exceed the reported input total. - const inputTokens = toResponsesUsageNumber( - value.prompt_tokens ?? - toResponsesUsageNumber(value.prompt_cache_miss_tokens) + - cachedTokens + - cacheCreationTokens, - ); - - return { - input_tokens: inputTokens, - input_tokens_details: { cached_tokens: cachedTokens }, - output_tokens: outputTokens, - output_tokens_details: { reasoning_tokens: reasoningTokens }, - total_tokens: - toResponsesUsageNumber(value.total_tokens) || inputTokens + outputTokens, - }; -}; - -const mapChatResponseToResponsesPayload = async ( - accessKeyId: string | null, - credentialFilename: string | null, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - upstreamPayload: Record, - serverToolExecutions: ServerToolExecution[], - imageExecutions: ImageGenerationExecution[] = [], -): Promise> => { - const responseId = createResponseId(); - const choices = Array.isArray(upstreamPayload.choices) - ? upstreamPayload.choices - : []; - const firstChoice = (choices[0] ?? {}) as { - message?: ChatResponseMessage; - }; - const toolCalls = Array.isArray(firstChoice.message?.tool_calls) - ? firstChoice.message.tool_calls - : []; - const outputText = stringifyContent(firstChoice.message?.content); - const createdAt = Math.floor(Date.now() / 1000); - const output: Array> = [ - ...serverToolExecutions.map((execution) => - buildResponsesWebSearchCallItem(execution, 'completed'), - ), - // Image generation is executed locally, so the standard - // `image_generation_call` item has to be synthesized here — the chat - // upstream has no notion of it. - ...imageExecutions.map((execution) => - buildResponsesImageGenerationCallItem(execution), - ), - ]; - const transcriptToolCalls = buildAssistantTranscriptToolCalls( - toolCalls, - defaults.tools, - ); - - // Emit the reasoning as its own item, ahead of the message it produced. - // - // Clients replay `output` verbatim on the next turn, so this is what lets a - // stateless Responses client carry reasoning forward. Without it the only - // reasoning we ever hand back is a transient `reasoning_text.delta`, which no - // client can replay because it has no id and no blob to send back. - // - // `encrypted_content` holds the reasoning verbatim, not ciphertext. Codex - // never opens it — it only echoes it — so plaintext round-trips exactly as - // well, and encrypting would obscure a value that carries no secret: the - // upstream gave us a summary, and the `summary` field below already shows it. - const reasoningText = - firstChoice.message?.reasoning_content ?? - firstChoice.message?.reasoning ?? - ''; - - if (reasoningText) { - output.push({ - id: createResponseReasoningId(), - type: 'reasoning', - summary: [{ type: 'summary_text', text: reasoningText }], - encrypted_content: `${REASONING_PREFIX}${reasoningText}`, - status: 'completed', - }); - } - - if (outputText || !toolCalls.length) { - output.push({ - id: createMessageId(), - type: 'message', - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: outputText, - annotations: [], - }, - ], - }); - } - - toolCalls.forEach((toolCall, index) => { - output.push( - buildResponsesToolCallOutputItem(defaults.tools, { - arguments: toolCall.function?.arguments ?? '', - callId: normalizeToolCallId(toolCall.id, index), - id: createResponseOutputId(), - name: toolCall.function?.name ?? 'function', - status: 'completed', - }), - ); - }); - - await storeResponseSession({ - accessKeyId, - credentialFilename, - createdAt: Date.now(), - id: responseId, - model, - transcript: [ - ...transcript, - { - role: 'assistant', - content: getAssistantTranscriptContent(outputText, transcriptToolCalls), - ...(transcriptToolCalls ? { tool_calls: transcriptToolCalls } : {}), - // A client that continues via `previous_response_id` rather than - // replaying `output` never sees the reasoning item, so the session is - // the only place the reasoning can survive into the next turn. - ...(reasoningText ? { reasoning: reasoningText } : {}), - }, - ], - defaults, - upstreamProtocol: 'chat', - }); - - return { - id: responseId, - object: 'response', - created_at: createdAt, - status: 'completed', - model, - output, - output_text: outputText, - usage: mapChatUsageToResponses(upstreamPayload.usage), - metadata: defaults.metadata ?? {}, - previous_response_id: previousResponseId, - }; -}; - -const buildResponsesWebSearchCallItem = ( - execution: ServerToolExecution | ServerToolInvocation, - status: 'completed' | 'in_progress', - id = `ws_${crypto.randomUUID().replaceAll('-', '')}`, -): Record => ({ - id, - type: 'web_search_call', - status, - action: - execution.type === 'web_search' - ? { type: 'search', query: execution.input.query } - : { - type: 'open_page', - url: - 'result' in execution - ? (execution.result.url ?? execution.input.url) - : execution.input.url, - }, -}); - -/** - * Emits an already-buffered chat payload as a Responses SSE stream. - * - * Used when a request had to be buffered to inspect it — image generation is - * executed locally, so the call cannot be forwarded before it is seen. The - * client still asked for `stream: true`, so the buffered result is replayed as - * the same event sequence a live stream would have produced. - * - * The text is replayed as delta events rather than arriving whole in - * `response.completed`: a client that renders as it reads subscribes to deltas - * and would otherwise show nothing until the turn ends. - */ -const mapChatResponseToResponsesStream = async ( - upstreamPayload: Record, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - proxyContext: ProxyContext, - imageExecutions: ImageGenerationExecution[], - serverToolExecutions: ServerToolExecution[] = [], -): Promise => { - const payload = await mapChatResponseToResponsesPayload( - proxyContext.accessKeyId, - proxyContext.credentialFilename, - defaults, - transcript, - model, - previousResponseId, - upstreamPayload, - serverToolExecutions, - imageExecutions, - ); - // The mapper creates and persists the session id, so the stream has to reuse - // it: advertising a different one would leave a client unable to continue the - // turn, because nothing was stored under the id it was given. - const responseId = String(payload.id); - const output = payload.output as Array>; - const messageIndex = output.findIndex((item) => item.type === 'message'); - const messageItem = - messageIndex === -1 - ? null - : (output[messageIndex] as { - content?: Array<{ text?: string }>; - id?: string; - }); - const messageText = messageItem?.content?.[0]?.text ?? ''; - const otherItems = output - .map((item, output_index) => ({ item, output_index })) - .filter(({ output_index }) => output_index !== messageIndex); - - // The live path announces a server-tool item as in-progress and narrates its - // lifecycle before closing it, and consumers can subscribe to those events. - // A buffered replay that jumps straight to `done` hides the search entirely - // from a client watching for it. - const serverToolFrames = ({ - item, - output_index, - }: { - item: Record; - output_index: number; - }): Array> => { - const itemId = String(item.id ?? ''); - - if (item.type !== 'web_search_call') { - return [ - { - item, - output_index, - response_id: responseId, - type: 'response.output_item.added', - }, - ]; - } - - return [ - { - item: { ...item, status: 'in_progress' }, - output_index, - response_id: responseId, - type: 'response.output_item.added', - }, - { - item_id: itemId, - output_index, - type: 'response.web_search_call.in_progress', - }, - { - item_id: itemId, - output_index, - type: 'response.web_search_call.searching', - }, - { - item_id: itemId, - output_index, - type: 'response.web_search_call.completed', - }, - ]; - }; - - const frames: Array> = [ - { - response: { ...payload, output: [], status: 'in_progress' }, - type: 'response.created', - }, - { - response: { id: responseId, status: 'in_progress' }, - type: 'response.in_progress', - }, - ...otherItems.flatMap(({ item, output_index }) => - serverToolFrames({ item, output_index }), - ), - ...otherItems.map(({ item, output_index }) => ({ - item, - output_index, - response_id: responseId, - type: 'response.output_item.done', - })), - ]; - - // Mirrors the live path: the message item is announced, filled by deltas, - // then closed. No `content_part` events — the live path does not emit them. - if (messageItem && messageIndex !== -1) { - frames.push({ - item: { ...messageItem, status: 'in_progress' }, - output_index: messageIndex, - response_id: responseId, - type: 'response.output_item.added', - }); - - if (messageText) { - frames.push({ - delta: messageText, - item_id: messageItem.id, - output_index: messageIndex, - response_id: responseId, - type: 'response.output_text.delta', - }); - frames.push({ - item: messageItem, - output_index: messageIndex, - response_id: responseId, - text: messageText, - type: 'response.output_text.done', - }); - } - - frames.push({ - item: messageItem, - output_index: messageIndex, - response_id: responseId, - type: 'response.output_item.done', - }); - } - - frames.push({ - response: { ...payload, id: responseId }, - type: 'response.completed', - }); - - const body = [ - ...frames.map( - (frame) => `event: ${frame.type}\ndata: ${JSON.stringify(frame)}`, - ), - 'data: [DONE]', - '', - ].join('\n\n'); - - return new Response(body, { - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -}; - -const mapChatStreamToResponsesEventStream = ( - upstreamResponse: Response, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - proxyContext: ProxyContext, - responseId = createResponseId(), - providedServerToolItems?: ResponsesServerToolItem[], - emitOpeningEvents = true, - emitServerToolLifecycle = true, - providedOutputIndexAllocator?: () => number, - rejectErrorPayloads = false, -): Response => { - if (!upstreamResponse.ok || !upstreamResponse.body) { - return upstreamResponse; - } - - const serverToolItems = - providedServerToolItems ?? - getServerToolExecutions(upstreamResponse).map((execution, outputIndex) => { - const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; - - return { - completed: buildResponsesWebSearchCallItem(execution, 'completed', id), - inProgress: buildResponsesWebSearchCallItem( - execution, - 'in_progress', - id, - ), - outputIndex, - }; - }); - let outputText = ''; - // Reasoning accumulated from stream deltas. The delta events alone are not - // replayable — a client needs a reasoning item in the completed output, with - // a blob of its own, to send anything back on the next turn. - let streamedReasoning = ''; - // Claimed on the first reasoning delta, which lands before any text, so the - // item sorts ahead of the message it produced. - let reasoningOutputIndex: number | null = null; - let reasoningItemAdded = false; - // Fixed when the first reasoning delta arrives, so the `output_item.added` - // event and the completed output reference the same id. - let reasoningItemId = ''; - - let nextOutputIndex = - serverToolItems.reduce( - (maximum, item) => Math.max(maximum, item.outputIndex), - -1, - ) + 1; - const allocateOutputIndex = - providedOutputIndexAllocator ?? (() => nextOutputIndex++); - const messageState: StreamingMessageState = { - outputIndex: null, - outputItemId: createMessageId(), - }; - let messageAddedEmitted = false; - const toolCallStates = new Map(); - const toolCallStateKeys = new Map(); - let latestUsage: unknown = null; - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - const closer = createStreamCloser(); - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - - const stream = new ReadableStream({ - start: (controller) => { - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - let totalToolArgumentLength = 0; - let streamRejected = false; - - const enqueueEvent = (payload: Record): void => { - const eventType = - typeof payload.type === 'string' ? payload.type : 'message'; - controller.enqueue( - encoder.encode( - `event: ${eventType}\ndata: ${JSON.stringify(payload)}\n\n`, - ), - ); - }; - - const buildStreamingMessageItem = ( - status: 'completed' | 'in_progress', - ): Record => ({ - id: messageState.outputItemId, - type: 'message', - role: 'assistant', - status, - content: [ - { - type: 'output_text', - text: outputText, - annotations: [], - }, - ], - }); - - // Carries the reasoning verbatim rather than encrypted — same reasoning as - // the non-streaming item above. - const buildStreamingReasoningItem = (): Record => ({ - id: reasoningItemId, - type: 'reasoning', - summary: [{ type: 'summary_text', text: streamedReasoning }], - encrypted_content: `${REASONING_PREFIX}${streamedReasoning}`, - status: 'completed', - }); - - const ensureReasoningItemAdded = (): void => { - if (reasoningItemAdded) { - return; - } - - reasoningOutputIndex ??= allocateOutputIndex(); - enqueueEvent({ - type: 'response.output_item.added', - item: buildStreamingReasoningItem(), - output_index: reasoningOutputIndex, - response_id: responseId, - }); - reasoningItemAdded = true; - }; - - const ensureMessageAdded = (): void => { - if (messageAddedEmitted) { - return; - } - - messageState.outputIndex ??= allocateOutputIndex(); - enqueueEvent({ - type: 'response.output_item.added', - item: buildStreamingMessageItem('in_progress'), - output_index: messageState.outputIndex, - response_id: responseId, - }); - messageAddedEmitted = true; - }; - - if (emitOpeningEvents) { - enqueueEvent({ - type: 'response.created', - response: { - id: responseId, - object: 'response', - created_at: Math.floor(Date.now() / 1000), - model, - output: [], - }, - }); - enqueueEvent({ - type: 'response.in_progress', - response: { - id: responseId, - status: 'in_progress', - }, - }); - } - if (emitServerToolLifecycle) { - serverToolItems.forEach(({ completed, inProgress, outputIndex }) => { - const itemId = String(inProgress.id); - enqueueEvent({ - type: 'response.output_item.added', - item: inProgress, - output_index: outputIndex, - response_id: responseId, - }); - enqueueEvent({ - type: 'response.web_search_call.in_progress', - item_id: itemId, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.web_search_call.searching', - item_id: itemId, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.web_search_call.completed', - item_id: itemId, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.output_item.done', - item: completed, - output_index: outputIndex, - response_id: responseId, - }); - }); - } - - const maybeEmitToolCallAdded = ( - toolCallState: StreamingToolCallState, - allowIncompleteName = false, - ): void => { - if (toolCallState.addedEmitted) { - return; - } - - const shouldWaitForInitialName = - !allowIncompleteName && - Boolean(defaults.tools?.length) && - toolCallState.name.length === 0; - const shouldWaitForMoreName = - !allowIncompleteName && - defaults.tools?.length && - toolCallState.name.length > 0 && - hasSupportedLongerToolNamePrefix(defaults.tools, toolCallState.name); - - if (shouldWaitForInitialName || shouldWaitForMoreName) { - return; - } - - enqueueEvent({ - type: 'response.output_item.added', - item: buildResponsesToolCallOutputItem(defaults.tools, { - arguments: '', - callId: toolCallState.callId, - id: toolCallState.outputItemId, - name: toolCallState.name || 'function', - status: 'in_progress', - }), - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - - toolCallState.addedEmitted = true; - toolCallState.pendingArgumentDeltas.forEach((delta) => { - enqueueEvent({ - type: getResponsesToolCallArgumentDeltaEventType( - defaults.tools, - toolCallState.name, - ), - delta, - item_id: toolCallState.outputItemId, - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - }); - toolCallState.pendingArgumentDeltas = []; - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - const transcriptToolCalls = - buildStreamingAssistantTranscriptToolCalls( - [...toolCallStates.values()], - defaults.tools, - ); - try { - await storeResponseSession({ - accessKeyId: proxyContext.accessKeyId, - credentialFilename: proxyContext.credentialFilename, - createdAt: Date.now(), - id: responseId, - model, - transcript: [ - ...transcript, - { - role: 'assistant', - content: getAssistantTranscriptContent( - outputText, - transcriptToolCalls, - ), - ...(transcriptToolCalls - ? { tool_calls: transcriptToolCalls } - : {}), - // Same reason as the non-streaming path: a client that - // continues via `previous_response_id` instead of - // replaying `output` would otherwise lose the reasoning. - ...(streamedReasoning - ? { reasoning: streamedReasoning } - : {}), - }, - ], - defaults, - upstreamProtocol: 'chat', - }); - } catch (error) { - console.error( - '[CodeBuddy2API] Failed to persist Responses session', - error, - ); - enqueueEvent({ - type: 'response.error', - error: { message: 'Failed to persist response session' }, - }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - releaseReader(); - controller.close(); - return; - } - [...toolCallStates.values()].forEach((toolCallState) => { - maybeEmitToolCallAdded(toolCallState, true); - enqueueEvent({ - type: 'response.output_item.done', - item: buildResponsesToolCallOutputItem(defaults.tools, { - arguments: toolCallState.arguments, - callId: toolCallState.callId, - id: toolCallState.outputItemId, - name: toolCallState.name || 'function', - status: 'completed', - }), - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - enqueueEvent({ - type: getResponsesToolCallArgumentDeltaEventType( - defaults.tools, - toolCallState.name, - ).replace('.delta', '.done'), - arguments: toolCallState.arguments, - item_id: toolCallState.outputItemId, - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - }); - if (outputText) { - ensureMessageAdded(); - enqueueEvent({ - type: 'response.output_text.done', - item: buildStreamingMessageItem('completed'), - output_index: messageState.outputIndex, - response_id: responseId, - text: outputText, - }); - enqueueEvent({ - type: 'response.output_item.done', - item: buildStreamingMessageItem('completed'), - output_index: messageState.outputIndex, - response_id: responseId, - }); - } - enqueueEvent({ - type: 'response.completed', - response: { - id: responseId, - status: 'completed', - output_text: outputText, - previous_response_id: previousResponseId, - usage: mapChatUsageToResponses(latestUsage), - output: [ - ...serverToolItems.map(({ completed, outputIndex }) => ({ - item: completed, - outputIndex, - })), - // Streamed reasoning needs the same replayable item the - // non-streaming path emits. It sorts ahead of the message by - // taking the next index before the message claims its own — - // the deltas come first on the wire, so the item order has - // to match or a client replaying `output` scrambles it. - ...(streamedReasoning && reasoningOutputIndex !== null - ? [ - { - item: buildStreamingReasoningItem(), - outputIndex: reasoningOutputIndex, - }, - ] - : []), - ...(outputText && messageState.outputIndex !== null - ? [ - { - item: buildStreamingMessageItem('completed'), - outputIndex: messageState.outputIndex, - }, - ] - : []), - ...[...toolCallStates.values()].map((toolCallState) => ({ - item: buildResponsesToolCallOutputItem(defaults.tools, { - arguments: toolCallState.arguments, - callId: toolCallState.callId, - id: toolCallState.outputItemId, - name: toolCallState.name || 'function', - status: 'completed', - }), - outputIndex: toolCallState.outputIndex, - })), - ] - .sort((left, right) => left.outputIndex - right.outputIndex) - .map(({ item }) => item), - }, - }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - releaseReader(); - controller.close(); - return; - } - - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_BUFFER_LENGTH) { - buffer = ''; - } - - for (const frame of frames) { - if (streamRejected) { - break; - } - if (frame.length > MAX_STREAM_BUFFER_LENGTH) { - continue; - } - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - continue; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - // The upstream here is the chat pipeline, which reports a deadline - // as a terminal error chunk and closes cleanly. Surfacing it keeps - // the client from seeing an empty successful response. - const upstreamError = readTimeoutFrame(frame); - - if (upstreamError !== null) { - streamRejected = true; - enqueueEvent({ - type: 'response.error', - error: { message: upstreamError }, - }); - break; - } - - try { - const payload = JSON.parse(raw) as { - choices?: Array<{ - delta?: { - content?: string; - reasoning_content?: string; - tool_calls?: ChatResponseToolCall[]; - }; - }>; - error?: unknown; - usage?: unknown; - }; - if (rejectErrorPayloads && payload.error) { - const error = - typeof payload.error === 'object' - ? (payload.error as { message?: unknown }) - : null; - streamRejected = true; - enqueueEvent({ - type: 'response.error', - error: { - message: - typeof error?.message === 'string' - ? error.message - : typeof payload.error === 'string' - ? payload.error - : 'Upstream request failed', - }, - }); - break; - } - // The final upstream chunk carries the aggregated usage, so - // remember it for the downstream response.completed event. - if (payload.usage !== undefined) { - latestUsage = payload.usage; - } - const delta = payload.choices?.[0]?.delta; - - if (delta?.content) { - ensureMessageAdded(); - outputText = `${outputText}${delta.content}`; - if (outputText.length > MAX_STREAM_TEXT_LENGTH) { - throw new Error('Response output exceeds the maximum size'); - } - enqueueEvent({ - type: 'response.output_text.delta', - delta: delta.content, - // Send only the item reference: embedding the accumulated - // text re-serializes it on every delta, which makes the - // enqueued volume quadratic in the output size. The full - // text still arrives intact in response.output_text.done - // and response.completed. - item_id: messageState.outputItemId, - output_index: messageState.outputIndex, - response_id: responseId, - }); - } - - if (delta?.reasoning_content) { - reasoningItemId ||= createResponseReasoningId(); - streamedReasoning += delta.reasoning_content; - ensureReasoningItemAdded(); - enqueueEvent({ - type: 'response.reasoning_text.delta', - delta: delta.reasoning_content, - response_id: responseId, - }); - } - - delta?.tool_calls?.forEach((toolCall, position) => { - const lookupKeys = getStreamingToolCallLookupKeys( - toolCall, - position, - ); - const existingCanonicalKey = lookupKeys - .map((key) => toolCallStateKeys.get(key) ?? key) - .find((key) => toolCallStates.has(key)); - const canonicalKey = - existingCanonicalKey ?? - getStreamingToolCallCanonicalKey(toolCall, position); - const existing = toolCallStates.get(canonicalKey); - const outputIndex = existing - ? existing.outputIndex - : allocateOutputIndex(); - const current = existing ?? { - addedEmitted: false, - arguments: '', - canonicalKey, - callId: normalizeToolCallId(toolCall.id, outputIndex), - name: '', - outputIndex, - outputItemId: createResponseOutputId(), - pendingArgumentDeltas: [], - }; - - if (toolCall.function?.name) { - if ( - current.name.length + toolCall.function.name.length > - MAX_TOOL_NAME_LENGTH - ) { - throw new Error( - 'Response tool name exceeds the maximum size', - ); - } - current.name += toolCall.function.name; - } - maybeEmitToolCallAdded(current); - - if (toolCall.function?.arguments) { - if ( - current.arguments.length + - toolCall.function.arguments.length > - MAX_TOOL_ARGUMENT_LENGTH - ) { - throw new Error( - 'Response tool arguments exceed the maximum size', - ); - } - if ( - totalToolArgumentLength + - toolCall.function.arguments.length > - MAX_RESPONSE_SESSION_TOTAL_BYTES - ) { - throw new Error( - 'Response tool arguments exceed the maximum size', - ); - } - totalToolArgumentLength += toolCall.function.arguments.length; - current.arguments = `${current.arguments}${toolCall.function.arguments}`; - if (current.addedEmitted) { - enqueueEvent({ - type: getResponsesToolCallArgumentDeltaEventType( - defaults.tools, - current.name, - ), - delta: toolCall.function.arguments, - item_id: current.outputItemId, - output_index: current.outputIndex, - response_id: responseId, - }); - } else { - current.pendingArgumentDeltas.push( - toolCall.function.arguments, - ); - } - } - - toolCallStates.set(canonicalKey, current); - lookupKeys.forEach((key) => { - toolCallStateKeys.set(key, current.canonicalKey); - }); - }); - } catch (error) { - if ( - error instanceof Error && - error.message.includes('maximum size') - ) { - streamRejected = true; - } - console.error( - '[CodeBuddy2API] Failed to parse upstream SSE frame', - { - route: '/v1/responses', - frame: raw.slice(0, 1000), - }, - ); - enqueueEvent({ - type: 'response.error', - error: { - message: 'Failed to parse upstream SSE frame', - }, - }); - } - } - - if (streamRejected) { - try { - await reader!.cancel(); - } finally { - releaseReader(); - controller.close(); - } - return; - } - } - }; - - void pump().catch((error) => { - if (cancelled) return; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage === null) { - closer.mark(); - controller.error(error); - return; - } - - streamRejected = true; - void reader?.cancel().then( - () => undefined, - () => undefined, - ); - releaseReader(); - closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - releaseReader(); - } - }, - }); - - return new Response(stream, { - status: 200, - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -}; - -const createResponsesEventStream = async ( - request: NextRequest, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - maxOutputTokens: number | undefined, - proxyContext: ProxyContext, - debugTrace?: DebugTrace, -): Promise => { - const translatedTools = translateResponsesToolsToChat(defaults.tools); - const translatedToolNames = new Set( - ( - (translatedTools ?? []) as Array<{ - function: { name: string }; - }> - ).map((tool) => normalizeToolName(tool.function.name)), - ); - const [searchEnabled, fetchEnabled] = await Promise.all([ - translatedToolNames.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) - ? isWebSearchEnabled() - : false, - translatedToolNames.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) - ? isWebFetchEnabled() - : false, - ]); - - const chatBody = { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }; - - // Image generation is executed locally, so a streaming request has to be - // buffered first to see whether the model asked for an image. Without this - // the call is forwarded as an ordinary function_call the client is expected - // to resolve — and nothing would ever generate the image. - // - // Handled before the server-tool branch below: a turn may declare both, and - // gating on search/fetch would silently skip generation whenever those were - // enabled. - if (hasImageGenerationTool(defaults.tools)) { - const { executions, response, serverToolExecutions } = - await executeImageGenerationLoop({ - body: chatBody, - // Buffered so the tool call can be inspected before any delta reaches - // the client; the ordinary path below stays live. - callUpstream: (loopBody) => - proxyChatCompletions( - request, - { ...loopBody, stream: false } as never, - proxyContext, - debugTrace, - '/v1/responses', - ), - context: proxyContext, - request, - }); - - // Always consumed, even when nothing was generated: the loop has already - // sent the turn upstream, and re-issuing it would bill twice and could - // return a different answer than the one inspected. - if (!response.ok) { - return response; - } - - return mapChatResponseToResponsesStream( - (await response.json()) as Record, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - executions, - serverToolExecutions, - ); - } - - if (!searchEnabled && !fetchEnabled) { - const upstreamResponse = await proxyChatCompletions( - request, - chatBody as never, - proxyContext, - debugTrace, - '/v1/responses', - ); - - return mapChatStreamToResponsesEventStream( - upstreamResponse, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - ); - } - - const encoder = new TextEncoder(); - const responseId = createResponseId(); - const serverToolItems: ResponsesServerToolItem[] = []; - const itemsByInvocationId = new Map(); - let nextOutputIndex = 0; - const allocateOutputIndex = (): number => nextOutputIndex++; - let activeReader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - - const stream = new ReadableStream({ - start: (controller) => { - const enqueueEvent = ( - payload: Record & { type: string }, - ): void => { - if (cancelled) return; - controller.enqueue( - encoder.encode( - `event: ${payload.type}\ndata: ${JSON.stringify(payload)}\n\n`, - ), - ); - }; - - enqueueEvent({ - type: 'response.created', - response: { - id: responseId, - object: 'response', - created_at: Math.floor(Date.now() / 1000), - model, - output: [], - }, - }); - enqueueEvent({ - type: 'response.in_progress', - response: { id: responseId, status: 'in_progress' }, - }); - - const run = async (): Promise => { - const upstreamResponse = await proxyChatCompletions( - request, - { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames( - transcript, - defaults.tools, - ), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }, - proxyContext, - debugTrace, - '/v1/responses', - { - emitStreamEvents: true, - onCall: (invocation) => { - const outputIndex = allocateOutputIndex(); - const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; - const item = { - completed: buildResponsesWebSearchCallItem( - invocation, - 'completed', - id, - ), - inProgress: buildResponsesWebSearchCallItem( - invocation, - 'in_progress', - id, - ), - outputIndex, - }; - serverToolItems.push(item); - itemsByInvocationId.set(invocation.id, item); - enqueueEvent({ - type: 'response.output_item.added', - item: item.inProgress, - output_index: outputIndex, - response_id: responseId, - }); - enqueueEvent({ - type: 'response.web_search_call.in_progress', - item_id: id, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.web_search_call.searching', - item_id: id, - output_index: outputIndex, - }); - }, - onResult: (execution) => { - const item = itemsByInvocationId.get(execution.id)!; - const id = String(item.inProgress.id); - item.completed = buildResponsesWebSearchCallItem( - execution, - 'completed', - id, - ); - enqueueEvent({ - type: 'response.web_search_call.completed', - item_id: id, - output_index: item.outputIndex, - }); - enqueueEvent({ - type: 'response.output_item.done', - item: item.completed, - output_index: item.outputIndex, - response_id: responseId, - }); - }, - }, - ); - - if (cancelled) { - await upstreamResponse.body?.cancel(); - return; - } - - if (!upstreamResponse.ok || !upstreamResponse.body) { - enqueueEvent({ - type: 'response.error', - error: { message: 'Upstream request failed' }, - }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - controller.close(); - return; - } - - const mappedResponse = mapChatStreamToResponsesEventStream( - upstreamResponse, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - responseId, - serverToolItems, - false, - false, - allocateOutputIndex, - true, - ); - const reader = mappedResponse.body!.getReader(); - activeReader = reader; - - while (true) { - const { done, value } = await reader.read(); - if (cancelled) return; - if (done) break; - controller.enqueue(value); - } - - reader.releaseLock(); - activeReader = null; - controller.close(); - }; +// --------------------------------------------------------------------------- +// /v1/responses route entry point +// +// The machinery this route relies on lives in the sibling modules: +// +// responses-types shared type declarations +// responses-session session persistence +// responses-tools Responses <-> Chat tool translation +// responses-transcript transcript construction and usage mapping +// responses-payload non-streaming payload assembly +// responses-stream SSE stream mapping +// --------------------------------------------------------------------------- - void run().catch((error) => { - if (!cancelled) controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - await activeReader?.cancel(reason); - activeReader?.releaseLock(); - activeReader = null; - }, - }); +import type { NextRequest } from 'next/server'; - return new Response(stream, { - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -}; +import { getDefaultModel } from '../domain/config'; +import { getCredentialSupportedModels } from '../domain/credentials'; +import type { DebugTrace } from '../domain/debug'; +import { createErrorResponse } from '../shared/http'; +import { resolveRequestAccessKey } from './auth'; +import { + proxyChatCompletions, + proxyResponsesUpstream, + resolveProxyContext, + resolveProxyContextByCredentialFilename, +} from './codebuddy'; +import { executeImageGenerationLoop } from './image-generation'; +import { mapChatResponseToResponsesPayload } from './responses/payload'; +import { + getResponseSession, + getValidatedPreviousSession, + storeUpstreamResponseBinding, +} from './responses/session'; +import { createResponsesEventStream } from './responses/event-stream'; +import { + getResponsesCompatibilityError, + hasImageGenerationTool, + normalizeTranscriptMessageToolNames, + translateResponsesToolsToChat, + translateResponsesToolChoiceToChatWithTools, +} from './responses/tools'; +import { prepareTranscript } from './responses/transcript'; +import type { ResponsesRequestBody } from './responses/types'; +import { getServerToolExecutions } from './web-search-loop'; export const handleResponsesRequest = async ( request: NextRequest, @@ -2996,8 +283,5 @@ export const handleResponsesRequest = async ( } }; -export const resetResponseSessions = (): void => { - getSessionStore().clear(); - getSessionByteStore().clear(); - setSessionTotalBytes(0); -}; +export { translateResponsesToolsToChat } from './responses/tools'; +export { resetResponseSessions } from './responses/session'; diff --git a/lib/server/proxy/responses/event-stream.ts b/lib/server/proxy/responses/event-stream.ts new file mode 100644 index 0000000..0e3a3ba --- /dev/null +++ b/lib/server/proxy/responses/event-stream.ts @@ -0,0 +1,327 @@ +// --------------------------------------------------------------------------- +// Responses streaming orchestration +// +// Chooses how a Responses request is served — delegated to the image +// generation loop, bridged as a live server-tool stream, or mapped one chat +// SSE chunk at a time — and owns the session persistence for the result. +// --------------------------------------------------------------------------- + +import type { NextRequest } from 'next/server'; + +import { isWebFetchEnabled, isWebSearchEnabled } from '../../domain/config'; +import type { DebugTrace } from '../../domain/debug'; +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../../search/tool'; +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; +import { proxyChatCompletions, type ProxyContext } from '../codebuddy'; +import { executeImageGenerationLoop } from '../image-generation'; +import { + buildResponsesWebSearchCallItem, + mapChatResponseToResponsesStream, +} from './payload'; +import { createResponseId } from './ids'; + +import { mapChatStreamToResponsesEventStream } from './stream'; +import { + hasImageGenerationTool, + normalizeTranscriptMessageToolNames, + translateResponsesToolsToChat, + translateResponsesToolChoiceToChatWithTools, +} from './tools'; +import type { + ResponsesServerToolItem, + ResponseSessionDefaults, + TranscriptMessage, +} from './types'; + +export const createResponsesEventStream = async ( + request: NextRequest, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + maxOutputTokens: number | undefined, + proxyContext: ProxyContext, + debugTrace?: DebugTrace, +): Promise => { + const translatedTools = translateResponsesToolsToChat(defaults.tools); + const translatedToolNames = new Set( + ( + (translatedTools ?? []) as Array<{ + function: { name: string }; + }> + ).map((tool) => normalizeToolName(tool.function.name)), + ); + const [searchEnabled, fetchEnabled] = await Promise.all([ + translatedToolNames.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) + ? isWebSearchEnabled() + : false, + translatedToolNames.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) + ? isWebFetchEnabled() + : false, + ]); + + const chatBody = { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }; + + // Image generation is executed locally, so a streaming request has to be + // buffered first to see whether the model asked for an image. Without this + // the call is forwarded as an ordinary function_call the client is expected + // to resolve — and nothing would ever generate the image. + // + // Handled before the server-tool branch below: a turn may declare both, and + // gating on search/fetch would silently skip generation whenever those were + // enabled. + if (hasImageGenerationTool(defaults.tools)) { + const { executions, response, serverToolExecutions } = + await executeImageGenerationLoop({ + body: chatBody, + // Buffered so the tool call can be inspected before any delta reaches + // the client; the ordinary path below stays live. + callUpstream: (loopBody) => + proxyChatCompletions( + request, + { ...loopBody, stream: false } as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); + + // Always consumed, even when nothing was generated: the loop has already + // sent the turn upstream, and re-issuing it would bill twice and could + // return a different answer than the one inspected. + if (!response.ok) { + return response; + } + + return mapChatResponseToResponsesStream( + (await response.json()) as Record, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + executions, + serverToolExecutions, + ); + } + + if (!searchEnabled && !fetchEnabled) { + const upstreamResponse = await proxyChatCompletions( + request, + chatBody as never, + proxyContext, + debugTrace, + '/v1/responses', + ); + + return mapChatStreamToResponsesEventStream( + upstreamResponse, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + ); + } + + const encoder = new TextEncoder(); + const responseId = createResponseId(); + const serverToolItems: ResponsesServerToolItem[] = []; + const itemsByInvocationId = new Map(); + let nextOutputIndex = 0; + const allocateOutputIndex = (): number => nextOutputIndex++; + let activeReader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + + const stream = new ReadableStream({ + start: (controller) => { + const enqueueEvent = ( + payload: Record & { type: string }, + ): void => { + if (cancelled) return; + controller.enqueue( + encoder.encode( + `event: ${payload.type}\ndata: ${JSON.stringify(payload)}\n\n`, + ), + ); + }; + + enqueueEvent({ + type: 'response.created', + response: { + id: responseId, + object: 'response', + created_at: Math.floor(Date.now() / 1000), + model, + output: [], + }, + }); + enqueueEvent({ + type: 'response.in_progress', + response: { id: responseId, status: 'in_progress' }, + }); + + const run = async (): Promise => { + const upstreamResponse = await proxyChatCompletions( + request, + { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames( + transcript, + defaults.tools, + ), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }, + proxyContext, + debugTrace, + '/v1/responses', + { + emitStreamEvents: true, + onCall: (invocation) => { + const outputIndex = allocateOutputIndex(); + const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; + const item = { + completed: buildResponsesWebSearchCallItem( + invocation, + 'completed', + id, + ), + inProgress: buildResponsesWebSearchCallItem( + invocation, + 'in_progress', + id, + ), + outputIndex, + }; + serverToolItems.push(item); + itemsByInvocationId.set(invocation.id, item); + enqueueEvent({ + type: 'response.output_item.added', + item: item.inProgress, + output_index: outputIndex, + response_id: responseId, + }); + enqueueEvent({ + type: 'response.web_search_call.in_progress', + item_id: id, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.web_search_call.searching', + item_id: id, + output_index: outputIndex, + }); + }, + onResult: (execution) => { + const item = itemsByInvocationId.get(execution.id)!; + const id = String(item.inProgress.id); + item.completed = buildResponsesWebSearchCallItem( + execution, + 'completed', + id, + ); + enqueueEvent({ + type: 'response.web_search_call.completed', + item_id: id, + output_index: item.outputIndex, + }); + enqueueEvent({ + type: 'response.output_item.done', + item: item.completed, + output_index: item.outputIndex, + response_id: responseId, + }); + }, + }, + ); + + if (cancelled) { + await upstreamResponse.body?.cancel(); + return; + } + + if (!upstreamResponse.ok || !upstreamResponse.body) { + enqueueEvent({ + type: 'response.error', + error: { message: 'Upstream request failed' }, + }); + controller.enqueue(encodeDoneFrame()); + controller.close(); + return; + } + + const mappedResponse = mapChatStreamToResponsesEventStream( + upstreamResponse, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + responseId, + serverToolItems, + false, + false, + allocateOutputIndex, + true, + ); + const reader = mappedResponse.body!.getReader(); + activeReader = reader; + + while (true) { + const { done, value } = await reader.read(); + if (cancelled) return; + if (done) break; + controller.enqueue(value); + } + + reader.releaseLock(); + activeReader = null; + controller.close(); + }; + + void run().catch((error) => { + if (!cancelled) controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + await activeReader?.cancel(reason); + activeReader?.releaseLock(); + activeReader = null; + }, + }); + + return createSseResponse(stream); +}; diff --git a/lib/server/proxy/responses/ids.ts b/lib/server/proxy/responses/ids.ts new file mode 100644 index 0000000..52150ac --- /dev/null +++ b/lib/server/proxy/responses/ids.ts @@ -0,0 +1,40 @@ +/** + * Identifier minting for Responses objects. + * + * Every id is a short protocol prefix plus a UUID, which is what the Responses + * API expects and what clients key their own bookkeeping off. They live apart + * from the payload builder because the transcript layer needs them too, and + * reaching back into the payload for an id would make the two circular. + */ + +export const createResponseId = (): string => { + return `resp_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const createMessageId = (): string => { + return `msg_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const createResponseReasoningId = (): string => { + return `rs_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const createResponseOutputId = (): string => { + return `fc_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +/** + * Rewrites an Anthropic-style tool-call id into the `call_` shape OpenAI + * clients expect. An id the upstream already minted is returned untouched, so + * only synthetic ids are normalised. + */ +export const normalizeToolCallId = ( + id: string | undefined, + index: number, +): string => { + if (id && !id.startsWith('tooluse_')) { + return id; + } + + return `call_${id?.replace(/^tooluse_/, '') ?? index + 1}`; +}; diff --git a/lib/server/proxy/responses/payload.ts b/lib/server/proxy/responses/payload.ts new file mode 100644 index 0000000..c738d54 --- /dev/null +++ b/lib/server/proxy/responses/payload.ts @@ -0,0 +1,357 @@ +// --------------------------------------------------------------------------- +// Non-streaming Responses payload assembly +// --------------------------------------------------------------------------- + +import { stringifyContent } from '../../shared/content'; +import { + createSseResponse, + DONE_FRAME_TEXT, + eventFrameText, +} from '../../shared/sse'; +import type { ProxyContext } from '../codebuddy'; +import { + buildResponsesImageGenerationCallItem, + type ImageGenerationExecution, +} from '../image-generation'; +import { + createMessageId, + createResponseId, + createResponseOutputId, + createResponseReasoningId, + normalizeToolCallId, +} from './ids'; +import { storeResponseSession } from './session'; +import { buildResponsesToolCallOutputItem } from './tools'; +import { + buildAssistantTranscriptToolCalls, + getAssistantTranscriptContent, + mapChatUsageToResponses, + REASONING_PREFIX, +} from './transcript'; +import type { + ChatResponseMessage, + ResponseSessionDefaults, + TranscriptMessage, +} from './types'; +import type { + ServerToolExecution, + ServerToolInvocation, +} from '../web-search-loop'; + +export const mapChatResponseToResponsesPayload = async ( + accessKeyId: string | null, + credentialFilename: string | null, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + upstreamPayload: Record, + serverToolExecutions: ServerToolExecution[], + imageExecutions: ImageGenerationExecution[] = [], +): Promise> => { + const responseId = createResponseId(); + const choices = Array.isArray(upstreamPayload.choices) + ? upstreamPayload.choices + : []; + const firstChoice = (choices[0] ?? {}) as { + message?: ChatResponseMessage; + }; + const toolCalls = Array.isArray(firstChoice.message?.tool_calls) + ? firstChoice.message.tool_calls + : []; + const outputText = stringifyContent(firstChoice.message?.content); + const createdAt = Math.floor(Date.now() / 1000); + const output: Array> = [ + ...serverToolExecutions.map((execution) => + buildResponsesWebSearchCallItem(execution, 'completed'), + ), + // Image generation is executed locally, so the standard + // `image_generation_call` item has to be synthesized here — the chat + // upstream has no notion of it. + ...imageExecutions.map((execution) => + buildResponsesImageGenerationCallItem(execution), + ), + ]; + const transcriptToolCalls = buildAssistantTranscriptToolCalls( + toolCalls, + defaults.tools, + ); + + // Emit the reasoning as its own item, ahead of the message it produced. + // + // Clients replay `output` verbatim on the next turn, so this is what lets a + // stateless Responses client carry reasoning forward. Without it the only + // reasoning we ever hand back is a transient `reasoning_text.delta`, which no + // client can replay because it has no id and no blob to send back. + // + // `encrypted_content` holds the reasoning verbatim, not ciphertext. Codex + // never opens it — it only echoes it — so plaintext round-trips exactly as + // well, and encrypting would obscure a value that carries no secret: the + // upstream gave us a summary, and the `summary` field below already shows it. + const reasoningText = + firstChoice.message?.reasoning_content ?? + firstChoice.message?.reasoning ?? + ''; + + if (reasoningText) { + output.push({ + id: createResponseReasoningId(), + type: 'reasoning', + summary: [{ type: 'summary_text', text: reasoningText }], + encrypted_content: `${REASONING_PREFIX}${reasoningText}`, + status: 'completed', + }); + } + + if (outputText || !toolCalls.length) { + output.push({ + id: createMessageId(), + type: 'message', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text: outputText, + annotations: [], + }, + ], + }); + } + + toolCalls.forEach((toolCall, index) => { + output.push( + buildResponsesToolCallOutputItem(defaults.tools, { + arguments: toolCall.function?.arguments ?? '', + callId: normalizeToolCallId(toolCall.id, index), + id: createResponseOutputId(), + name: toolCall.function?.name ?? 'function', + status: 'completed', + }), + ); + }); + + await storeResponseSession({ + accessKeyId, + credentialFilename, + createdAt: Date.now(), + id: responseId, + model, + transcript: [ + ...transcript, + { + role: 'assistant', + content: getAssistantTranscriptContent(outputText, transcriptToolCalls), + ...(transcriptToolCalls ? { tool_calls: transcriptToolCalls } : {}), + // A client that continues via `previous_response_id` rather than + // replaying `output` never sees the reasoning item, so the session is + // the only place the reasoning can survive into the next turn. + ...(reasoningText ? { reasoning: reasoningText } : {}), + }, + ], + defaults, + upstreamProtocol: 'chat', + }); + + return { + id: responseId, + object: 'response', + created_at: createdAt, + status: 'completed', + model, + output, + output_text: outputText, + usage: mapChatUsageToResponses(upstreamPayload.usage), + metadata: defaults.metadata ?? {}, + previous_response_id: previousResponseId, + }; +}; + +export const buildResponsesWebSearchCallItem = ( + execution: ServerToolExecution | ServerToolInvocation, + status: 'completed' | 'in_progress', + id = `ws_${crypto.randomUUID().replaceAll('-', '')}`, +): Record => ({ + id, + type: 'web_search_call', + status, + action: + execution.type === 'web_search' + ? { type: 'search', query: execution.input.query } + : { + type: 'open_page', + url: + 'result' in execution + ? (execution.result.url ?? execution.input.url) + : execution.input.url, + }, +}); + +/** + * Emits an already-buffered chat payload as a Responses SSE stream. + * + * Used when a request had to be buffered to inspect it — image generation is + * executed locally, so the call cannot be forwarded before it is seen. The + * client still asked for `stream: true`, so the buffered result is replayed as + * the same event sequence a live stream would have produced. + * + * The text is replayed as delta events rather than arriving whole in + * `response.completed`: a client that renders as it reads subscribes to deltas + * and would otherwise show nothing until the turn ends. + */ +export const mapChatResponseToResponsesStream = async ( + upstreamPayload: Record, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + proxyContext: ProxyContext, + imageExecutions: ImageGenerationExecution[], + serverToolExecutions: ServerToolExecution[] = [], +): Promise => { + const payload = await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + defaults, + transcript, + model, + previousResponseId, + upstreamPayload, + serverToolExecutions, + imageExecutions, + ); + // The mapper creates and persists the session id, so the stream has to reuse + // it: advertising a different one would leave a client unable to continue the + // turn, because nothing was stored under the id it was given. + const responseId = String(payload.id); + const output = payload.output as Array>; + const messageIndex = output.findIndex((item) => item.type === 'message'); + const messageItem = + messageIndex === -1 + ? null + : (output[messageIndex] as { + content?: Array<{ text?: string }>; + id?: string; + }); + const messageText = messageItem?.content?.[0]?.text ?? ''; + const otherItems = output + .map((item, output_index) => ({ item, output_index })) + .filter(({ output_index }) => output_index !== messageIndex); + + // The live path announces a server-tool item as in-progress and narrates its + // lifecycle before closing it, and consumers can subscribe to those events. + // A buffered replay that jumps straight to `done` hides the search entirely + // from a client watching for it. + const serverToolFrames = ({ + item, + output_index, + }: { + item: Record; + output_index: number; + }): Array> => { + const itemId = String(item.id ?? ''); + + if (item.type !== 'web_search_call') { + return [ + { + item, + output_index, + response_id: responseId, + type: 'response.output_item.added', + }, + ]; + } + + return [ + { + item: { ...item, status: 'in_progress' }, + output_index, + response_id: responseId, + type: 'response.output_item.added', + }, + { + item_id: itemId, + output_index, + type: 'response.web_search_call.in_progress', + }, + { + item_id: itemId, + output_index, + type: 'response.web_search_call.searching', + }, + { + item_id: itemId, + output_index, + type: 'response.web_search_call.completed', + }, + ]; + }; + + const frames: Array> = [ + { + response: { ...payload, output: [], status: 'in_progress' }, + type: 'response.created', + }, + { + response: { id: responseId, status: 'in_progress' }, + type: 'response.in_progress', + }, + ...otherItems.flatMap(({ item, output_index }) => + serverToolFrames({ item, output_index }), + ), + ...otherItems.map(({ item, output_index }) => ({ + item, + output_index, + response_id: responseId, + type: 'response.output_item.done', + })), + ]; + + // Mirrors the live path: the message item is announced, filled by deltas, + // then closed. No `content_part` events — the live path does not emit them. + if (messageItem && messageIndex !== -1) { + frames.push({ + item: { ...messageItem, status: 'in_progress' }, + output_index: messageIndex, + response_id: responseId, + type: 'response.output_item.added', + }); + + if (messageText) { + frames.push({ + delta: messageText, + item_id: messageItem.id, + output_index: messageIndex, + response_id: responseId, + type: 'response.output_text.delta', + }); + frames.push({ + item: messageItem, + output_index: messageIndex, + response_id: responseId, + text: messageText, + type: 'response.output_text.done', + }); + } + + frames.push({ + item: messageItem, + output_index: messageIndex, + response_id: responseId, + type: 'response.output_item.done', + }); + } + + frames.push({ + response: { ...payload, id: responseId }, + type: 'response.completed', + }); + + const body = [ + ...frames.map((frame) => eventFrameText(String(frame.type), frame)), + DONE_FRAME_TEXT, + '', + ].join('\n\n'); + + return createSseResponse(body); +}; diff --git a/lib/server/proxy/responses/session.ts b/lib/server/proxy/responses/session.ts new file mode 100644 index 0000000..fd0d25e --- /dev/null +++ b/lib/server/proxy/responses/session.ts @@ -0,0 +1,227 @@ +// --------------------------------------------------------------------------- +// Response session persistence +// --------------------------------------------------------------------------- + +import type { ProxyContext } from '../codebuddy'; +import { + deleteStorageJson, + getStorageBackendMeta, + listStorageJson, + readStorageJson, + writeStorageJson, +} from '../../storage'; +import type { ResponseSession, ResponseSessionMetadata } from './types'; + +export const MAX_RESPONSE_SESSIONS = 1_000; +export const RESPONSE_SESSION_TTL_MS = 60 * 60 * 1000; +export const MAX_RESPONSE_SESSION_BYTES = 8 * 1024 * 1024; +export const MAX_RESPONSE_SESSION_TOTAL_BYTES = 64 * 1024 * 1024; +export const MAX_RESPONSE_TRANSCRIPT_MESSAGES = 200; +export const RESPONSE_SESSION_NAMESPACE = 'responses'; +export const RESPONSE_SESSION_INDEX_NAMESPACE = 'response-session-index'; + +const globalResponsesState = globalThis as typeof globalThis & { + __codebuddy2apiResponseSessions__?: Map; + __codebuddy2apiResponseSessionBytes__?: Map; + __codebuddy2apiResponseSessionTotalBytes__?: number; +}; + +export const getSessionStore = (): Map => { + if (!globalResponsesState.__codebuddy2apiResponseSessions__) { + globalResponsesState.__codebuddy2apiResponseSessions__ = new Map(); + } + + return globalResponsesState.__codebuddy2apiResponseSessions__; +}; + +export const getSessionByteStore = (): Map => { + if (!globalResponsesState.__codebuddy2apiResponseSessionBytes__) { + globalResponsesState.__codebuddy2apiResponseSessionBytes__ = new Map(); + } + + return globalResponsesState.__codebuddy2apiResponseSessionBytes__; +}; + +export const getSessionTotalBytes = (): number => { + return globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ ?? 0; +}; + +export const setSessionTotalBytes = (value: number): void => { + globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ = value; +}; + +export const removeLocalResponseSession = (id: string): void => { + const byteStore = getSessionByteStore(); + const store = getSessionStore(); + const bytes = byteStore.get(id) ?? 0; + store.delete(id); + byteStore.delete(id); + setSessionTotalBytes(Math.max(0, getSessionTotalBytes() - bytes)); +}; + +export const pruneResponseSessions = (): void => { + const store = getSessionStore(); + const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; + + for (const [id, session] of store) { + if (session.createdAt <= expiresBefore) { + removeLocalResponseSession(id); + } + } + + while ( + store.size > MAX_RESPONSE_SESSIONS || + getSessionTotalBytes() > MAX_RESPONSE_SESSION_TOTAL_BYTES + ) { + const oldestId = store.keys().next().value; + + // Guard against a byte total that has drifted out of step with the map. + // Without this, an empty map with a positive total makes the removal a + // no-op and spins here forever, blocking the event loop. + if (oldestId === undefined) { + setSessionTotalBytes(0); + break; + } + + removeLocalResponseSession(oldestId); + } +}; + +export const prunePgResponseSessions = async (): Promise => { + const metadataDocuments = await listStorageJson( + RESPONSE_SESSION_INDEX_NAMESPACE, + ); + const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; + const candidates = metadataDocuments + .map((document) => ({ key: document.key, ...document.value })) + .sort((left, right) => left.createdAt - right.createdAt); + const toDelete = candidates.filter( + (candidate) => candidate.createdAt <= expiresBefore, + ); + const remaining = candidates.filter( + (candidate) => candidate.createdAt > expiresBefore, + ); + let totalBytes = remaining.reduce( + (total, candidate) => total + candidate.bytes, + 0, + ); + + while ( + remaining.length > MAX_RESPONSE_SESSIONS || + totalBytes > MAX_RESPONSE_SESSION_TOTAL_BYTES + ) { + const candidate = remaining.shift()!; + toDelete.push(candidate); + totalBytes -= candidate.bytes; + } + + await Promise.all( + toDelete.flatMap((candidate) => [ + deleteStorageJson(RESPONSE_SESSION_NAMESPACE, candidate.key), + deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, candidate.key), + ]), + ); +}; + +export const isPgResponseSessionStore = (): boolean => { + return getStorageBackendMeta().backend === 'pg'; +}; + +export const getResponseSession = async ( + id: string, +): Promise => { + if (isPgResponseSessionStore()) { + const session = await readStorageJson( + RESPONSE_SESSION_NAMESPACE, + id, + ); + if (!session || session.createdAt <= Date.now() - RESPONSE_SESSION_TTL_MS) { + if (session) { + await deleteStorageJson(RESPONSE_SESSION_NAMESPACE, id); + await deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, id); + } + return undefined; + } + return session; + } + + pruneResponseSessions(); + return getSessionStore().get(id); +}; + +export const getValidatedPreviousSession = async ( + previousResponseId: string | null, + accessKeyId: string | null, +): Promise => { + const previousSession = previousResponseId + ? await getResponseSession(previousResponseId) + : undefined; + + if ( + previousResponseId && + (!previousSession || previousSession.accessKeyId !== accessKeyId) + ) { + throw new Error('Unknown or expired previous_response_id'); + } + + return previousSession; +}; + +export const storeResponseSession = async ( + session: ResponseSession, +): Promise => { + const serialized = JSON.stringify(session); + if (Buffer.byteLength(serialized, 'utf8') > MAX_RESPONSE_SESSION_BYTES) { + throw new Error('Response session exceeds the maximum size'); + } + + if (isPgResponseSessionStore()) { + await writeStorageJson(RESPONSE_SESSION_NAMESPACE, session.id, session); + await writeStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, session.id, { + bytes: Buffer.byteLength(serialized, 'utf8'), + createdAt: session.createdAt, + }); + try { + await prunePgResponseSessions(); + } catch (error) { + console.warn('[CodeBuddy2API] Unable to prune Responses sessions', error); + } + return; + } + + const store = getSessionStore(); + const byteStore = getSessionByteStore(); + const previousBytes = byteStore.get(session.id) ?? 0; + const sessionBytes = Buffer.byteLength(serialized, 'utf8'); + store.set(session.id, session); + byteStore.set(session.id, sessionBytes); + setSessionTotalBytes(getSessionTotalBytes() - previousBytes + sessionBytes); + pruneResponseSessions(); +}; + +export const storeUpstreamResponseBinding = async ({ + model, + proxyContext, + responseId, +}: { + model: string; + proxyContext: ProxyContext; + responseId: string; +}): Promise => { + await storeResponseSession({ + accessKeyId: proxyContext.accessKeyId, + credentialFilename: proxyContext.credentialFilename, + createdAt: Date.now(), + defaults: {}, + id: responseId, + model, + transcript: [], + upstreamProtocol: 'responses', + }); +}; + +export const resetResponseSessions = (): void => { + getSessionStore().clear(); + getSessionByteStore().clear(); + setSessionTotalBytes(0); +}; diff --git a/lib/server/proxy/responses/stream.ts b/lib/server/proxy/responses/stream.ts new file mode 100644 index 0000000..1e490db --- /dev/null +++ b/lib/server/proxy/responses/stream.ts @@ -0,0 +1,713 @@ +// --------------------------------------------------------------------------- +// Responses SSE stream mapping +// --------------------------------------------------------------------------- + +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; +import { + createStreamCloser, + readTimeoutFrame, + responsesStreamErrorChunks, + toUpstreamTimeoutMessage, +} from '../../shared/upstream-timeout'; +import type { ProxyContext } from '../codebuddy'; + +import { storeResponseSession } from './session'; +import { buildResponsesWebSearchCallItem } from './payload'; +import { + createMessageId, + createResponseId, + createResponseOutputId, + createResponseReasoningId, + normalizeToolCallId, +} from './ids'; +import { + buildResponsesToolCallOutputItem, + getResponsesToolCallArgumentDeltaEventType, + hasSupportedLongerToolNamePrefix, +} from './tools'; +import { + buildStreamingAssistantTranscriptToolCalls, + getAssistantTranscriptContent, + getStreamingToolCallCanonicalKey, + getStreamingToolCallLookupKeys, + mapChatUsageToResponses, + REASONING_PREFIX, +} from './transcript'; +import type { + ChatResponseToolCall, + ResponsesServerToolItem, + ResponseSessionDefaults, + StreamingMessageState, + StreamingToolCallState, + TranscriptMessage, +} from './types'; +import { getServerToolExecutions } from '../web-search-loop'; +import { MAX_RESPONSE_SESSION_TOTAL_BYTES } from './session'; + +const MAX_STREAM_BUFFER_LENGTH = 1_000_000; +const MAX_STREAM_TEXT_LENGTH = 2_000_000; +const MAX_TOOL_ARGUMENT_LENGTH = 1_000_000; +const MAX_TOOL_NAME_LENGTH = 256; + +export const mapChatStreamToResponsesEventStream = ( + upstreamResponse: Response, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + proxyContext: ProxyContext, + responseId = createResponseId(), + providedServerToolItems?: ResponsesServerToolItem[], + emitOpeningEvents = true, + emitServerToolLifecycle = true, + providedOutputIndexAllocator?: () => number, + rejectErrorPayloads = false, +): Response => { + if (!upstreamResponse.ok || !upstreamResponse.body) { + return upstreamResponse; + } + + const serverToolItems = + providedServerToolItems ?? + getServerToolExecutions(upstreamResponse).map((execution, outputIndex) => { + const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; + + return { + completed: buildResponsesWebSearchCallItem(execution, 'completed', id), + inProgress: buildResponsesWebSearchCallItem( + execution, + 'in_progress', + id, + ), + outputIndex, + }; + }); + let outputText = ''; + // Reasoning accumulated from stream deltas. The delta events alone are not + // replayable — a client needs a reasoning item in the completed output, with + // a blob of its own, to send anything back on the next turn. + let streamedReasoning = ''; + // Claimed on the first reasoning delta, which lands before any text, so the + // item sorts ahead of the message it produced. + let reasoningOutputIndex: number | null = null; + let reasoningItemAdded = false; + // Fixed when the first reasoning delta arrives, so the `output_item.added` + // event and the completed output reference the same id. + let reasoningItemId = ''; + + let nextOutputIndex = + serverToolItems.reduce( + (maximum, item) => Math.max(maximum, item.outputIndex), + -1, + ) + 1; + const allocateOutputIndex = + providedOutputIndexAllocator ?? (() => nextOutputIndex++); + const messageState: StreamingMessageState = { + outputIndex: null, + outputItemId: createMessageId(), + }; + let messageAddedEmitted = false; + const toolCallStates = new Map(); + const toolCallStateKeys = new Map(); + let latestUsage: unknown = null; + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + const closer = createStreamCloser(); + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + + const stream = new ReadableStream({ + start: (controller) => { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + let totalToolArgumentLength = 0; + let streamRejected = false; + + const enqueueEvent = (payload: Record): void => { + const eventType = + typeof payload.type === 'string' ? payload.type : 'message'; + controller.enqueue( + encoder.encode( + `event: ${eventType}\ndata: ${JSON.stringify(payload)}\n\n`, + ), + ); + }; + + const buildStreamingMessageItem = ( + status: 'completed' | 'in_progress', + ): Record => ({ + id: messageState.outputItemId, + type: 'message', + role: 'assistant', + status, + content: [ + { + type: 'output_text', + text: outputText, + annotations: [], + }, + ], + }); + + // Carries the reasoning verbatim rather than encrypted — same reasoning as + // the non-streaming item above. + const buildStreamingReasoningItem = (): Record => ({ + id: reasoningItemId, + type: 'reasoning', + summary: [{ type: 'summary_text', text: streamedReasoning }], + encrypted_content: `${REASONING_PREFIX}${streamedReasoning}`, + status: 'completed', + }); + + const ensureReasoningItemAdded = (): void => { + if (reasoningItemAdded) { + return; + } + + reasoningOutputIndex ??= allocateOutputIndex(); + enqueueEvent({ + type: 'response.output_item.added', + item: buildStreamingReasoningItem(), + output_index: reasoningOutputIndex, + response_id: responseId, + }); + reasoningItemAdded = true; + }; + + const ensureMessageAdded = (): void => { + if (messageAddedEmitted) { + return; + } + + messageState.outputIndex ??= allocateOutputIndex(); + enqueueEvent({ + type: 'response.output_item.added', + item: buildStreamingMessageItem('in_progress'), + output_index: messageState.outputIndex, + response_id: responseId, + }); + messageAddedEmitted = true; + }; + + if (emitOpeningEvents) { + enqueueEvent({ + type: 'response.created', + response: { + id: responseId, + object: 'response', + created_at: Math.floor(Date.now() / 1000), + model, + output: [], + }, + }); + enqueueEvent({ + type: 'response.in_progress', + response: { + id: responseId, + status: 'in_progress', + }, + }); + } + if (emitServerToolLifecycle) { + serverToolItems.forEach(({ completed, inProgress, outputIndex }) => { + const itemId = String(inProgress.id); + enqueueEvent({ + type: 'response.output_item.added', + item: inProgress, + output_index: outputIndex, + response_id: responseId, + }); + enqueueEvent({ + type: 'response.web_search_call.in_progress', + item_id: itemId, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.web_search_call.searching', + item_id: itemId, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.web_search_call.completed', + item_id: itemId, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.output_item.done', + item: completed, + output_index: outputIndex, + response_id: responseId, + }); + }); + } + + const maybeEmitToolCallAdded = ( + toolCallState: StreamingToolCallState, + allowIncompleteName = false, + ): void => { + if (toolCallState.addedEmitted) { + return; + } + + const shouldWaitForInitialName = + !allowIncompleteName && + Boolean(defaults.tools?.length) && + toolCallState.name.length === 0; + const shouldWaitForMoreName = + !allowIncompleteName && + defaults.tools?.length && + toolCallState.name.length > 0 && + hasSupportedLongerToolNamePrefix(defaults.tools, toolCallState.name); + + if (shouldWaitForInitialName || shouldWaitForMoreName) { + return; + } + + enqueueEvent({ + type: 'response.output_item.added', + item: buildResponsesToolCallOutputItem(defaults.tools, { + arguments: '', + callId: toolCallState.callId, + id: toolCallState.outputItemId, + name: toolCallState.name || 'function', + status: 'in_progress', + }), + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + + toolCallState.addedEmitted = true; + toolCallState.pendingArgumentDeltas.forEach((delta) => { + enqueueEvent({ + type: getResponsesToolCallArgumentDeltaEventType( + defaults.tools, + toolCallState.name, + ), + delta, + item_id: toolCallState.outputItemId, + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + }); + toolCallState.pendingArgumentDeltas = []; + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + const transcriptToolCalls = + buildStreamingAssistantTranscriptToolCalls( + [...toolCallStates.values()], + defaults.tools, + ); + try { + await storeResponseSession({ + accessKeyId: proxyContext.accessKeyId, + credentialFilename: proxyContext.credentialFilename, + createdAt: Date.now(), + id: responseId, + model, + transcript: [ + ...transcript, + { + role: 'assistant', + content: getAssistantTranscriptContent( + outputText, + transcriptToolCalls, + ), + ...(transcriptToolCalls + ? { tool_calls: transcriptToolCalls } + : {}), + // Same reason as the non-streaming path: a client that + // continues via `previous_response_id` instead of + // replaying `output` would otherwise lose the reasoning. + ...(streamedReasoning + ? { reasoning: streamedReasoning } + : {}), + }, + ], + defaults, + upstreamProtocol: 'chat', + }); + } catch (error) { + console.error( + '[CodeBuddy2API] Failed to persist Responses session', + error, + ); + enqueueEvent({ + type: 'response.error', + error: { message: 'Failed to persist response session' }, + }); + controller.enqueue(encodeDoneFrame()); + releaseReader(); + controller.close(); + return; + } + [...toolCallStates.values()].forEach((toolCallState) => { + maybeEmitToolCallAdded(toolCallState, true); + enqueueEvent({ + type: 'response.output_item.done', + item: buildResponsesToolCallOutputItem(defaults.tools, { + arguments: toolCallState.arguments, + callId: toolCallState.callId, + id: toolCallState.outputItemId, + name: toolCallState.name || 'function', + status: 'completed', + }), + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + enqueueEvent({ + type: getResponsesToolCallArgumentDeltaEventType( + defaults.tools, + toolCallState.name, + ).replace('.delta', '.done'), + arguments: toolCallState.arguments, + item_id: toolCallState.outputItemId, + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + }); + if (outputText) { + ensureMessageAdded(); + enqueueEvent({ + type: 'response.output_text.done', + item: buildStreamingMessageItem('completed'), + output_index: messageState.outputIndex, + response_id: responseId, + text: outputText, + }); + enqueueEvent({ + type: 'response.output_item.done', + item: buildStreamingMessageItem('completed'), + output_index: messageState.outputIndex, + response_id: responseId, + }); + } + enqueueEvent({ + type: 'response.completed', + response: { + id: responseId, + status: 'completed', + output_text: outputText, + previous_response_id: previousResponseId, + usage: mapChatUsageToResponses(latestUsage), + output: [ + ...serverToolItems.map(({ completed, outputIndex }) => ({ + item: completed, + outputIndex, + })), + // Streamed reasoning needs the same replayable item the + // non-streaming path emits. It sorts ahead of the message by + // taking the next index before the message claims its own — + // the deltas come first on the wire, so the item order has + // to match or a client replaying `output` scrambles it. + ...(streamedReasoning && reasoningOutputIndex !== null + ? [ + { + item: buildStreamingReasoningItem(), + outputIndex: reasoningOutputIndex, + }, + ] + : []), + ...(outputText && messageState.outputIndex !== null + ? [ + { + item: buildStreamingMessageItem('completed'), + outputIndex: messageState.outputIndex, + }, + ] + : []), + ...[...toolCallStates.values()].map((toolCallState) => ({ + item: buildResponsesToolCallOutputItem(defaults.tools, { + arguments: toolCallState.arguments, + callId: toolCallState.callId, + id: toolCallState.outputItemId, + name: toolCallState.name || 'function', + status: 'completed', + }), + outputIndex: toolCallState.outputIndex, + })), + ] + .sort((left, right) => left.outputIndex - right.outputIndex) + .map(({ item }) => item), + }, + }); + controller.enqueue(encodeDoneFrame()); + releaseReader(); + controller.close(); + return; + } + + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_BUFFER_LENGTH) { + buffer = ''; + } + + for (const frame of frames) { + if (streamRejected) { + break; + } + if (frame.length > MAX_STREAM_BUFFER_LENGTH) { + continue; + } + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + continue; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + // The upstream here is the chat pipeline, which reports a deadline + // as a terminal error chunk and closes cleanly. Surfacing it keeps + // the client from seeing an empty successful response. + const upstreamError = readTimeoutFrame(frame); + + if (upstreamError !== null) { + streamRejected = true; + enqueueEvent({ + type: 'response.error', + error: { message: upstreamError }, + }); + break; + } + + try { + const payload = JSON.parse(raw) as { + choices?: Array<{ + delta?: { + content?: string; + reasoning_content?: string; + tool_calls?: ChatResponseToolCall[]; + }; + }>; + error?: unknown; + usage?: unknown; + }; + if (rejectErrorPayloads && payload.error) { + const error = + typeof payload.error === 'object' + ? (payload.error as { message?: unknown }) + : null; + streamRejected = true; + enqueueEvent({ + type: 'response.error', + error: { + message: + typeof error?.message === 'string' + ? error.message + : typeof payload.error === 'string' + ? payload.error + : 'Upstream request failed', + }, + }); + break; + } + // The final upstream chunk carries the aggregated usage, so + // remember it for the downstream response.completed event. + if (payload.usage !== undefined) { + latestUsage = payload.usage; + } + const delta = payload.choices?.[0]?.delta; + + if (delta?.content) { + ensureMessageAdded(); + outputText = `${outputText}${delta.content}`; + if (outputText.length > MAX_STREAM_TEXT_LENGTH) { + throw new Error('Response output exceeds the maximum size'); + } + enqueueEvent({ + type: 'response.output_text.delta', + delta: delta.content, + // Send only the item reference: embedding the accumulated + // text re-serializes it on every delta, which makes the + // enqueued volume quadratic in the output size. The full + // text still arrives intact in response.output_text.done + // and response.completed. + item_id: messageState.outputItemId, + output_index: messageState.outputIndex, + response_id: responseId, + }); + } + + if (delta?.reasoning_content) { + reasoningItemId ||= createResponseReasoningId(); + streamedReasoning += delta.reasoning_content; + ensureReasoningItemAdded(); + enqueueEvent({ + type: 'response.reasoning_text.delta', + delta: delta.reasoning_content, + response_id: responseId, + }); + } + + delta?.tool_calls?.forEach((toolCall, position) => { + const lookupKeys = getStreamingToolCallLookupKeys( + toolCall, + position, + ); + const existingCanonicalKey = lookupKeys + .map((key) => toolCallStateKeys.get(key) ?? key) + .find((key) => toolCallStates.has(key)); + const canonicalKey = + existingCanonicalKey ?? + getStreamingToolCallCanonicalKey(toolCall, position); + const existing = toolCallStates.get(canonicalKey); + const outputIndex = existing + ? existing.outputIndex + : allocateOutputIndex(); + const current = existing ?? { + addedEmitted: false, + arguments: '', + canonicalKey, + callId: normalizeToolCallId(toolCall.id, outputIndex), + name: '', + outputIndex, + outputItemId: createResponseOutputId(), + pendingArgumentDeltas: [], + }; + + if (toolCall.function?.name) { + if ( + current.name.length + toolCall.function.name.length > + MAX_TOOL_NAME_LENGTH + ) { + throw new Error( + 'Response tool name exceeds the maximum size', + ); + } + current.name += toolCall.function.name; + } + maybeEmitToolCallAdded(current); + + if (toolCall.function?.arguments) { + if ( + current.arguments.length + + toolCall.function.arguments.length > + MAX_TOOL_ARGUMENT_LENGTH + ) { + throw new Error( + 'Response tool arguments exceed the maximum size', + ); + } + if ( + totalToolArgumentLength + + toolCall.function.arguments.length > + MAX_RESPONSE_SESSION_TOTAL_BYTES + ) { + throw new Error( + 'Response tool arguments exceed the maximum size', + ); + } + totalToolArgumentLength += toolCall.function.arguments.length; + current.arguments = `${current.arguments}${toolCall.function.arguments}`; + if (current.addedEmitted) { + enqueueEvent({ + type: getResponsesToolCallArgumentDeltaEventType( + defaults.tools, + current.name, + ), + delta: toolCall.function.arguments, + item_id: current.outputItemId, + output_index: current.outputIndex, + response_id: responseId, + }); + } else { + current.pendingArgumentDeltas.push( + toolCall.function.arguments, + ); + } + } + + toolCallStates.set(canonicalKey, current); + lookupKeys.forEach((key) => { + toolCallStateKeys.set(key, current.canonicalKey); + }); + }); + } catch (error) { + if ( + error instanceof Error && + error.message.includes('maximum size') + ) { + streamRejected = true; + } + console.error( + '[CodeBuddy2API] Failed to parse upstream SSE frame', + { + route: '/v1/responses', + frame: raw.slice(0, 1000), + }, + ); + enqueueEvent({ + type: 'response.error', + error: { + message: 'Failed to parse upstream SSE frame', + }, + }); + } + } + + if (streamRejected) { + try { + await reader!.cancel(); + } finally { + releaseReader(); + controller.close(); + } + return; + } + } + }; + + void pump().catch((error) => { + if (cancelled) return; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage === null) { + closer.mark(); + controller.error(error); + return; + } + + streamRejected = true; + void reader?.cancel().then( + () => undefined, + () => undefined, + ); + releaseReader(); + closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + releaseReader(); + } + }, + }); + + return createSseResponse(stream, { status: 200 }); +}; diff --git a/lib/server/proxy/responses/tools.ts b/lib/server/proxy/responses/tools.ts new file mode 100644 index 0000000..0124d22 --- /dev/null +++ b/lib/server/proxy/responses/tools.ts @@ -0,0 +1,582 @@ +// --------------------------------------------------------------------------- +// Responses <-> Chat tool translation +// --------------------------------------------------------------------------- + +import { createErrorResponse } from '../../shared/http'; +import { + buildWebFetchToolDefinition, + buildWebSearchToolDefinition, + markServerTool, + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_FETCH_TOOL_TYPE_PREFIX, + WEB_SEARCH_TOOL_NAME, + WEB_SEARCH_TOOL_TYPE_PREFIX, +} from '../../search/tool'; +import { + buildImageGenerationChatTool, + IMAGE_GENERATION_CHAT_TOOL_NAME, + IMAGE_GENERATION_TOOL_TYPE, +} from '../image-generation'; +import type { + ResponsesRequestBody, + SupportedChatTool, + SupportedResponsesTool, + TranscriptMessage, +} from './types'; + +export const TOOL_SEARCH_PROXY_NAME = 'tool_search'; +export const CUSTOM_TOOL_INPUT_FIELD = 'input'; +export const CUSTOM_TOOL_INPUT_DESCRIPTION = + 'Raw string input for the original custom tool.'; + +export const flattenNamespaceToolName = ( + namespace: string, + name: string, +): string => { + return `${namespace}__${name}`; +}; + +export const extractFunctionDefinition = ( + tool: Record, +): Record | null => { + const nested = + typeof tool.function === 'object' && tool.function !== null + ? (tool.function as Record) + : {}; + + const name = nested.name ?? tool.name; + if (typeof name !== 'string' || name.length === 0) { + return null; + } + + const functionDef: Record = { name }; + + const description = nested.description ?? tool.description; + if (description !== undefined) { + functionDef.description = description; + } + + const parameters = nested.parameters ?? tool.parameters; + if (parameters !== undefined) { + functionDef.parameters = parameters; + } + + const strict = nested.strict ?? tool.strict; + if (strict !== undefined) { + functionDef.strict = strict; + } + + return functionDef; +}; + +export const buildCustomToolDefinition = ( + tool: Record, +): Record | null => { + const name = typeof tool.name === 'string' ? tool.name.trim() : ''; + + if (!name) { + return null; + } + + const description = + typeof tool.description === 'string' && tool.description.trim() + ? tool.description + : `Custom tool ${name}`; + + return { + name, + description, + parameters: { + type: 'object', + properties: { + [CUSTOM_TOOL_INPUT_FIELD]: { + type: 'string', + description: CUSTOM_TOOL_INPUT_DESCRIPTION, + }, + }, + required: [CUSTOM_TOOL_INPUT_FIELD], + }, + }; +}; + +export const buildToolSearchDefinition = (): Record => { + return { + name: TOOL_SEARCH_PROXY_NAME, + description: + 'Search and load Codex tools, plugins, connectors, and MCP namespaces for the current task.', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'Search query for tools or connectors to load.', + }, + limit: { + type: 'integer', + description: 'Maximum number of tool groups to return.', + }, + }, + required: ['query'], + }, + }; +}; + +export const toSupportedChatTool = ( + tool: SupportedResponsesTool, + namespace?: string, +): SupportedChatTool[] => { + const toolType = typeof tool.type === 'string' ? tool.type : 'function'; + + // Server-side search and fetch carry no function schema, so the generic + // branch below drops them. Emit them as functions unconditionally and let + // the proxy loop resolve the configured backend asynchronously. SearXNG + // needs local configuration, while CodeBuddy search does not. + if ( + normalizeToolName(toolType).startsWith( + normalizeToolName(WEB_SEARCH_TOOL_TYPE_PREFIX), + ) + ) { + const definition = buildWebSearchToolDefinition(); + + return [ + { + chatName: WEB_SEARCH_TOOL_NAME, + kind: 'function', + originalName: WEB_SEARCH_TOOL_NAME, + serverDeclared: true, + tool: definition, + }, + ]; + } + + // Fetch needs no deployment-level configuration — the local backend is always + // available and the CodeBuddy backend needs only a credential — so it is + // advertised unconditionally and gated later by the enable toggle. + if ( + normalizeToolName(toolType).startsWith( + normalizeToolName(WEB_FETCH_TOOL_TYPE_PREFIX), + ) + ) { + const definition = buildWebFetchToolDefinition(); + + return [ + { + chatName: WEB_FETCH_TOOL_NAME, + kind: 'function', + originalName: WEB_FETCH_TOOL_NAME, + serverDeclared: true, + tool: definition, + }, + ]; + } + + // Image generation has no chat-protocol equivalent, so the declaration is + // rewritten as a function and the call is executed by the proxy. It is + // advertised only on the chat path: the responses passthrough hands the + // native declaration straight to the upstream, which supports it. + if (toolType === IMAGE_GENERATION_TOOL_TYPE) { + return [ + { + chatName: IMAGE_GENERATION_CHAT_TOOL_NAME, + kind: 'function' as const, + originalName: IMAGE_GENERATION_TOOL_TYPE, + serverDeclared: true, + tool: buildImageGenerationChatTool(), + }, + ]; + } + + if (toolType === 'namespace') { + const namespaceName = typeof tool.name === 'string' ? tool.name.trim() : ''; + const children = ( + Array.isArray(tool.tools) + ? tool.tools + : Array.isArray(tool.children) + ? tool.children + : [] + ).filter((item): item is SupportedResponsesTool => { + return Boolean(item && typeof item === 'object'); + }); + + if (!namespaceName || !children.length) { + return []; + } + + return children.flatMap((child) => + toSupportedChatTool(child, namespaceName), + ); + } + + if (toolType === 'tool_search') { + const definition = buildToolSearchDefinition(); + return [ + { + chatName: TOOL_SEARCH_PROXY_NAME, + kind: 'tool_search', + originalName: TOOL_SEARCH_PROXY_NAME, + tool: definition, + }, + ]; + } + + if (toolType === 'custom') { + const definition = buildCustomToolDefinition(tool); + + if (!definition || typeof definition.name !== 'string') { + return []; + } + + return [ + { + chatName: definition.name, + kind: 'custom', + originalName: definition.name, + tool: definition, + }, + ]; + } + + const functionDef = extractFunctionDefinition(tool); + + if (!functionDef || typeof functionDef.name !== 'string') { + return []; + } + + const originalName = functionDef.name; + const chatName = namespace + ? flattenNamespaceToolName(namespace, originalName) + : toolType === 'mcp' && + typeof tool.server_label === 'string' && + tool.server_label.trim() + ? flattenNamespaceToolName(tool.server_label.trim(), originalName) + : originalName; + + return [ + { + chatName, + kind: toolType === 'mcp' ? 'mcp' : 'function', + namespace: + namespace || + (typeof tool.server_label === 'string' ? tool.server_label : undefined), + originalName, + serverLabel: + toolType === 'mcp' && typeof tool.server_label === 'string' + ? tool.server_label + : undefined, + tool: { + ...functionDef, + name: chatName, + }, + }, + ]; +}; + +/** + * True when the client declared an `image_generation` tool. Gating on this + * keeps the ordinary path free of an extra upstream round trip. + */ +export const hasImageGenerationTool = ( + tools: ResponsesRequestBody['tools'], +): boolean => { + return Boolean( + tools?.some( + (tool) => + typeof tool?.type === 'string' && + tool.type.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_TOOL_TYPE, + ), + ); +}; + +export const getSupportedChatTools = ( + tools: ResponsesRequestBody['tools'], +): SupportedChatTool[] => { + if (!tools?.length) { + return []; + } + + return tools.flatMap((tool) => toSupportedChatTool(tool)); +}; + +export const findSupportedToolByName = ( + tools: ResponsesRequestBody['tools'], + name: string, +): SupportedChatTool | null => { + if (!tools?.length || !name) { + return null; + } + + return ( + getSupportedChatTools(tools).find( + (tool) => tool.chatName === name || tool.originalName === name, + ) ?? null + ); +}; + +export const hasSupportedLongerToolNamePrefix = ( + tools: ResponsesRequestBody['tools'], + prefix: string, +): boolean => { + if (!tools?.length || !prefix) { + return false; + } + + return getSupportedChatTools(tools).some((tool) => { + const name = tool.chatName; + return ( + typeof name === 'string' && + name.length > prefix.length && + name.startsWith(prefix) + ); + }); +}; + +export const buildResponsesToolCallOutputItem = ( + tools: ResponsesRequestBody['tools'], + toolCall: { + arguments: string; + callId: string; + id: string; + name: string; + status: 'completed' | 'in_progress'; + }, +): Record => { + const originalTool = findSupportedToolByName(tools, toolCall.name); + const itemType = originalTool?.kind === 'mcp' ? 'mcp_call' : 'function_call'; + const item: Record = { + id: toolCall.id, + type: itemType, + call_id: toolCall.callId, + name: originalTool?.originalName ?? toolCall.name ?? 'function', + arguments: toolCall.arguments, + status: toolCall.status, + }; + + if (originalTool?.kind === 'mcp' && originalTool.serverLabel) { + item.server_label = originalTool.serverLabel; + } + + if (originalTool?.kind === 'function' && originalTool.namespace) { + item.namespace = originalTool.namespace; + } + + return item; +}; + +export const getResponsesToolCallArgumentDeltaEventType = ( + tools: ResponsesRequestBody['tools'], + name: string, +): + | 'response.function_call_arguments.delta' + | 'response.mcp_call_arguments.delta' => { + return findSupportedToolByName(tools, name)?.kind === 'mcp' + ? 'response.mcp_call_arguments.delta' + : 'response.function_call_arguments.delta'; +}; + +export const translateResponsesToolsToChat = ( + tools: ResponsesRequestBody['tools'], +): unknown[] | undefined => { + if (!tools?.length) { + return undefined; + } + + const supported = getSupportedChatTools(tools); + if (!supported.length) { + return undefined; + } + + return supported.map((tool) => { + return { + type: 'function', + function: tool.tool, + ...(tool.serverDeclared ? markServerTool({}) : {}), + }; + }); +}; + +export const translateResponsesToolChoiceToChat = ( + toolChoice: unknown, +): unknown => { + if (typeof toolChoice !== 'object' || toolChoice === null) { + return toolChoice; + } + + const choice = toolChoice as Record; + + if ( + choice.type === 'function' && + choice.function && + typeof choice.function === 'object' + ) { + return toolChoice; + } + + if ( + (choice.type === 'auto' || + choice.type === 'none' || + choice.type === 'required') && + typeof choice.type === 'string' + ) { + return choice.type; + } + + // Responses API selects a function by name: + // {type: 'function', name: 'fn'} -> chat schema {type: 'function', function: {name: 'fn'}} + if (typeof choice.name === 'string') { + return { + type: 'function', + function: { name: choice.name }, + }; + } + + return toolChoice; +}; + +export const translateResponsesToolChoiceToChatWithTools = ( + tools: ResponsesRequestBody['tools'], + toolChoice: unknown, +): unknown => { + const translated = translateResponsesToolChoiceToChat(toolChoice); + + if (typeof translated !== 'object' || translated === null) { + return translated; + } + + const choice = translated as Record; + + if ( + choice.type === 'function' && + typeof choice.function === 'object' && + choice.function !== null + ) { + const functionChoice = choice.function as Record; + if (typeof functionChoice.name === 'string') { + return { + ...choice, + function: { + ...functionChoice, + name: resolveChatToolName(tools, functionChoice.name), + }, + }; + } + } + + return translated; +}; + +export const getNamedToolChoice = (toolChoice: unknown): string | null => { + if (typeof toolChoice !== 'object' || toolChoice === null) { + return null; + } + + const choice = toolChoice as Record; + + if (typeof choice.name === 'string' && choice.name.length > 0) { + return choice.name; + } + + if ( + choice.type === 'function' && + typeof choice.function === 'object' && + choice.function !== null && + typeof (choice.function as Record).name === 'string' + ) { + return (choice.function as Record).name; + } + + return null; +}; + +export const getResponsesCompatibilityError = ( + tools: ResponsesRequestBody['tools'], + toolChoice: unknown, +): Response | null => { + const supportedTools = getSupportedChatTools(tools); + + if (toolChoice === 'required' && supportedTools.length === 0) { + return createErrorResponse( + 400, + 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', + ); + } + + if (typeof toolChoice === 'object' && toolChoice !== null) { + const choice = toolChoice as Record; + const isPretranslatedFunctionChoice = + choice.type === 'function' && + typeof choice.function === 'object' && + choice.function !== null; + const isSimpleChoiceType = + choice.type === 'auto' || + choice.type === 'none' || + choice.type === 'required'; + const isNamedFunctionLikeChoice = typeof choice.name === 'string'; + + if ( + !isPretranslatedFunctionChoice && + !isSimpleChoiceType && + !isNamedFunctionLikeChoice + ) { + return createErrorResponse( + 400, + 'Unsupported Responses tool_choice for this /v1/responses adapter', + ); + } + + if (choice.type === 'required' && supportedTools.length === 0) { + return createErrorResponse( + 400, + 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', + ); + } + } + + const namedToolChoice = getNamedToolChoice(toolChoice); + if (namedToolChoice) { + const supportedNames = new Set( + supportedTools + .map((tool) => tool.originalName) + .filter((name): name is string => typeof name === 'string'), + ); + + if (!supportedNames.has(namedToolChoice)) { + return createErrorResponse( + 400, + 'tool_choice references a tool that is not available to this /v1/responses adapter', + ); + } + } + + return null; +}; + +export const resolveChatToolName = ( + tools: ResponsesRequestBody['tools'], + name: string, +): string => { + return findSupportedToolByName(tools, name)?.chatName ?? name; +}; + +export const normalizeTranscriptMessageToolNames = ( + transcript: TranscriptMessage[], + tools: ResponsesRequestBody['tools'], +): TranscriptMessage[] => { + return transcript.map((message) => { + if (!message.tool_calls?.length) { + return message; + } + + return { + ...message, + tool_calls: message.tool_calls.map((toolCall) => ({ + ...toolCall, + function: { + ...toolCall.function, + name: resolveChatToolName(tools, toolCall.function.name), + }, + })), + }; + }); +}; diff --git a/lib/server/proxy/responses/transcript.ts b/lib/server/proxy/responses/transcript.ts new file mode 100644 index 0000000..9744f6c --- /dev/null +++ b/lib/server/proxy/responses/transcript.ts @@ -0,0 +1,459 @@ +// --------------------------------------------------------------------------- +// Transcript construction and usage mapping +// --------------------------------------------------------------------------- + +import { getDefaultModel } from '../../domain/config'; +import { stringifyContent } from '../../shared/content'; +import { extractImageUrl, isImageContentPart } from '../codebuddy'; +import { createResponseOutputId, normalizeToolCallId } from './ids'; +import { + getValidatedPreviousSession, + MAX_RESPONSE_TRANSCRIPT_MESSAGES, +} from './session'; +import { findSupportedToolByName } from './tools'; +import type { + ChatContentPart, + ChatResponseToolCall, + ResponsesInputItem, + ResponsesRequestBody, + ResponseSession, + ResponseSessionDefaults, + StreamingToolCallState, + TranscriptContent, + TranscriptMessage, +} from './types'; + +export const buildAssistantTranscriptToolCalls = ( + toolCalls: ChatResponseToolCall[], + tools?: ResponsesRequestBody['tools'], +): TranscriptMessage['tool_calls'] | undefined => { + if (!toolCalls.length) { + return undefined; + } + + return toolCalls.map((toolCall, index) => ({ + id: normalizeToolCallId(toolCall.id, index), + type: 'function', + function: { + name: + findSupportedToolByName(tools, toolCall.function?.name ?? '') + ?.originalName ?? + toolCall.function?.name ?? + 'function', + arguments: toolCall.function?.arguments ?? '', + }, + })); +}; + +export const buildStreamingAssistantTranscriptToolCalls = ( + toolCallStates: StreamingToolCallState[], + tools?: ResponsesRequestBody['tools'], +): TranscriptMessage['tool_calls'] | undefined => { + if (!toolCallStates.length) { + return undefined; + } + + return toolCallStates.map((toolCallState) => ({ + id: toolCallState.callId, + type: 'function', + function: { + arguments: toolCallState.arguments, + name: + findSupportedToolByName(tools, toolCallState.name)?.originalName ?? + toolCallState.name, + }, + })); +}; + +export const getAssistantTranscriptContent = ( + outputText: string, + toolCalls: TranscriptMessage['tool_calls'] | undefined, +): string | null => { + return toolCalls?.length ? outputText || null : outputText; +}; + +/** + * Keeps image parts as structured content so the chat path can rebuild them + * upstream. Text parts are still flattened: the transcript is persisted across + * turns and replayed as Chat messages, and the Responses converter only + * recognises images in the OpenAI `image_url` shape. + */ +export const mapInputContentToTranscriptContent = ( + content: unknown, +): TranscriptContent | null => { + if (typeof content === 'string') { + return content; + } + + if (!Array.isArray(content)) { + return null; + } + + const parts = content.filter((part) => part !== null && part !== undefined); + + if (!parts.some(isImageContentPart)) { + return null; + } + + const mapped = parts.flatMap((part): ChatContentPart[] => { + if (typeof part === 'string') { + return [part]; + } + + if (isImageContentPart(part)) { + const imageUrl = extractImageUrl(part); + + return imageUrl + ? [{ image_url: { url: imageUrl }, type: 'image_url' }] + : []; + } + + if (part && typeof part === 'object' && 'text' in part) { + return [String((part as { text?: unknown }).text ?? '')]; + } + + return []; + }); + + return mapped.length ? mapped : null; +}; + +/** + * Marks an `encrypted_content` value we minted, so we can tell it apart from a + * blob issued by someone else. + * + * Not a security measure. Codex never opens this field — it only echoes it — so + * plaintext round-trips fine, but a marker is what stops us from reading a + * genuinely encrypted blob as if it were reasoning text. + */ +export const REASONING_PREFIX = 'cbreason1:'; + +/** + * Pulls readable reasoning out of a replayed `reasoning` item. + * + * Only values we minted are used: anything else — an OpenAI-issued blob, say — + * is opaque ciphertext, and forwarding it upstream would send gibberish where + * reasoning belongs. The summary is the fallback in that case. + * + * The Agents SDK sends summaries as `summary: [{type: 'summary_text', text}]`, + * so a client that never received our blob still gets its reasoning through. + */ +export const extractReasoningFromItem = (item: ResponsesInputItem): string => { + const blob = item.encrypted_content; + + if (typeof blob === 'string' && blob.startsWith(REASONING_PREFIX)) { + return blob.slice(REASONING_PREFIX.length); + } + + if (!Array.isArray(item.summary)) { + return ''; + } + + return item.summary + .map((entry) => { + if (typeof entry === 'string') { + return entry; + } + + if (entry && typeof entry === 'object' && 'text' in entry) { + return String((entry as { text?: unknown }).text ?? ''); + } + + return ''; + }) + .join(''); +}; + +export const mapInputItemToMessage = ( + item: ResponsesInputItem, +): TranscriptMessage | null => { + if (item.type === 'reasoning' || item.type === 'compaction') { + // Reasoning is not a message. Without this branch the item fell through to + // the plain-message case at the bottom, where it has neither `role` nor + // `content` — becoming an empty `{role:'user', content:''}` entry that the + // chat upstream sees as a turn the user never sent, repeated on every + // later turn of the conversation. + // + // Signal the reasoning back to the caller instead, which attaches it to the + // assistant message it accompanies. Returning `null` when there is nothing + // to recover keeps an empty reasoning item from emitting a message at all. + const reasoning = extractReasoningFromItem(item); + + return reasoning ? { role: 'assistant', content: null, reasoning } : null; + } + + if (item.type === 'function_call' || item.type === 'mcp_call') { + return { + role: 'assistant', + content: null, + tool_calls: [ + { + id: item.call_id ?? createResponseOutputId(), + type: 'function', + function: { + name: item.name ?? 'function', + arguments: item.arguments ?? '', + }, + }, + ], + }; + } + + if (item.type === 'function_call_output' || item.type === 'mcp_call_output') { + // A tool may return an image, e.g. a screenshot. Keep it structured so the + // Responses converter can rebuild it as an image; stringifying would hand + // the model the base64 payload as text. + const outputContent = + mapInputContentToTranscriptContent(item.output) ?? + stringifyContent(item.output); + + if (item.call_id) { + return { + role: 'tool', + content: outputContent, + tool_call_id: item.call_id, + }; + } + + return { + role: 'user', + content: outputContent, + }; + } + + if (item.type === 'mcp_approval_response') { + return { + role: 'user', + content: JSON.stringify(item), + }; + } + + // Every other item type returns above, so what is left is a plain message: + // either one with a declared `type: 'message'`, or one carrying only + // `role`/`content`. Images are kept structured so the chat path can rebuild + // them; a message without an image stays flattened. + const imageContent = mapInputContentToTranscriptContent(item.content); + + if (imageContent !== null) { + return { + role: item.role ?? 'user', + content: imageContent, + }; + } + + return { + role: item.role ?? 'user', + content: item.text ?? stringifyContent(item.content), + }; +}; + +export const getStreamingToolCallCanonicalKey = ( + toolCall: ChatResponseToolCall, + position: number, +): string => { + if (toolCall.id) { + return `id:${toolCall.id}`; + } + + if (typeof toolCall.index === 'number') { + return `index:${toolCall.index}`; + } + + return `position:${position}`; +}; + +export const getStreamingToolCallLookupKeys = ( + toolCall: ChatResponseToolCall, + position: number, +): string[] => { + if (toolCall.id || typeof toolCall.index === 'number') { + return [ + toolCall.id ? `id:${toolCall.id}` : null, + typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, + ].filter((key): key is string => key !== null); + } + + return [`position:${position}`]; +}; + +export const prepareTranscript = async ( + body: ResponsesRequestBody, + accessKeyId: string | null, + previousSession?: ResponseSession, +): Promise<{ + defaults: ResponseSessionDefaults; + model: string; + transcript: TranscriptMessage[]; + previousResponseId: string | null; +}> => { + const previousResponseId = body.previous_response_id ?? null; + const resolvedPreviousSession = + previousSession ?? + (await getValidatedPreviousSession(previousResponseId, accessKeyId)); + + const transcript = (resolvedPreviousSession?.transcript ?? []).slice( + -MAX_RESPONSE_TRANSCRIPT_MESSAGES, + ); + // Reasoning recovered from replayed reasoning items, awaiting the assistant + // message it belongs to. Declared here so it spans the whole input array. + let pendingReasoning = ''; + while (transcript[0]?.role === 'tool') { + transcript.shift(); + } + const model = + typeof body.model === 'string' && body.model.trim() + ? body.model + : (resolvedPreviousSession?.model ?? (await getDefaultModel())); + const additionalTools = Array.isArray(body.input) + ? body.input.flatMap((item) => + item?.type === 'additional_tools' && Array.isArray(item.tools) + ? item.tools + : [], + ) + : []; + const baseTools = body.tools ?? resolvedPreviousSession?.defaults.tools; + const requestTools = [...(baseTools ?? []), ...additionalTools]; + const defaults = { + instructions: + body.instructions ?? + resolvedPreviousSession?.defaults.instructions ?? + undefined, + metadata: + body.metadata ?? resolvedPreviousSession?.defaults.metadata ?? undefined, + tools: requestTools.length > 0 ? requestTools : baseTools, + tool_choice: + body.tool_choice ?? + resolvedPreviousSession?.defaults.tool_choice ?? + undefined, + }; + + if (body.messages?.length) { + body.messages.forEach((item) => { + transcript.push({ + role: item.role ?? 'user', + content: + mapInputContentToTranscriptContent(item.content) ?? + stringifyContent(item.content), + }); + }); + } else if (typeof body.input === 'string') { + transcript.push({ role: 'user', content: body.input }); + } else if (Array.isArray(body.input)) { + body.input.forEach((item) => { + if (item.type === 'additional_tools') return; + + const message = mapInputItemToMessage(item); + + if (!message) { + return; + } + + // A reasoning item yields a reasoning-only entry. Fold it into the next + // assistant message so the upstream sees the reasoning where it belongs + // — attached to the turn that produced it — instead of as a bare turn. + // Anything left unconsumed at the end is dropped: reasoning with no + // following assistant message has nothing to attach to. + if (message.reasoning && !message.content && !message.tool_calls) { + pendingReasoning += message.reasoning; + return; + } + + // Attach any reasoning carried forward from a preceding reasoning item. + // `message.reasoning` is only ever set by the mapper below — clients + // cannot send it, since `ResponsesInputItem` has no such field — so + // there is no pre-existing value to merge with. + if (pendingReasoning) { + message.reasoning = pendingReasoning; + pendingReasoning = ''; + } + + transcript.push(message); + }); + } + + return { + defaults, + model, + transcript, + previousResponseId, + }; +}; + +export const toResponsesUsageNumber = (value: unknown): number => { + const numeric = + typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); + + if (!Number.isFinite(numeric) || numeric < 0) { + return 0; + } + + return numeric; +}; + +export const mapChatUsageToResponses = ( + usage: unknown, +): Record => { + if (!usage || typeof usage !== 'object') { + return { + input_tokens: 0, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 0, + }; + } + + const value = usage as { + cache_creation_input_tokens?: unknown; + cache_read_input_tokens?: unknown; + completion_tokens?: unknown; + completion_tokens_details?: { reasoning_tokens?: unknown }; + completion_thinking_tokens?: unknown; + input_tokens_details?: { cached_tokens?: unknown }; + prompt_cache_hit_tokens?: unknown; + prompt_cache_miss_tokens?: unknown; + prompt_cache_write_tokens?: unknown; + prompt_tokens?: unknown; + prompt_tokens_details?: { + cache_creation_tokens?: unknown; + cached_tokens?: unknown; + }; + total_tokens?: unknown; + }; + const outputTokens = toResponsesUsageNumber(value.completion_tokens); + const cachedTokens = toResponsesUsageNumber( + value.prompt_tokens_details?.cached_tokens ?? + value.input_tokens_details?.cached_tokens ?? + value.cache_read_input_tokens ?? + value.prompt_cache_hit_tokens, + ); + const cacheCreationTokens = toResponsesUsageNumber( + value.prompt_tokens_details?.cache_creation_tokens ?? + value.cache_creation_input_tokens ?? + value.prompt_cache_write_tokens, + ); + const reasoningTokens = toResponsesUsageNumber( + value.completion_tokens_details?.reasoning_tokens ?? + value.completion_thinking_tokens, + ); + // Chat usage is the single source of truth for both shapes. Keep the + // Responses counters faithful to it so clients never see zeroed metrics. + // prompt_tokens already covers its cached and created subsets, so the + // split counters are only summed when prompt_tokens is missing. Otherwise + // cached tokens would exceed the reported input total. + const inputTokens = toResponsesUsageNumber( + value.prompt_tokens ?? + toResponsesUsageNumber(value.prompt_cache_miss_tokens) + + cachedTokens + + cacheCreationTokens, + ); + + return { + input_tokens: inputTokens, + input_tokens_details: { cached_tokens: cachedTokens }, + output_tokens: outputTokens, + output_tokens_details: { reasoning_tokens: reasoningTokens }, + total_tokens: + toResponsesUsageNumber(value.total_tokens) || inputTokens + outputTokens, + }; +}; diff --git a/lib/server/proxy/responses/types.ts b/lib/server/proxy/responses/types.ts new file mode 100644 index 0000000..83632de --- /dev/null +++ b/lib/server/proxy/responses/types.ts @@ -0,0 +1,166 @@ +// --------------------------------------------------------------------------- +// OpenAI Responses API types +// --------------------------------------------------------------------------- + +export interface ResponsesInputItem { + type?: string; + role?: string; + content?: unknown; + text?: string; + arguments?: string; + output?: unknown; + name?: string; + call_id?: string; + tools?: Array<{ type?: string; name?: string } & Record>; + /** + * Present on `reasoning` items a client replays from an earlier response. + * We put the reasoning here verbatim; clients echo it back untouched. + * A compaction item carries the same field. + */ + encrypted_content?: string; + /** + * Reasoning summaries. The Agents SDK sends these back as + * `summary: [{type: 'summary_text', text}]`. + */ + summary?: unknown; +} + +export interface SupportedChatTool { + chatName: string; + kind: 'custom' | 'function' | 'mcp' | 'tool_search'; + namespace?: string; + originalName: string; + /** + * True when the client declared this as a provider-executed server tool + * (`web_search_20260209`, `web_fetch_20250910`, `web_search_preview`) rather + * than as its own function. + * + * Translation turns both into ordinary functions for upstream, so without this + * the proxy cannot tell them apart later — and the difference decides whether a + * tool that cannot be executed is dropped or forwarded. + */ + serverDeclared?: boolean; + serverLabel?: string; + tool: Record; +} + +export interface ResponsesRequestBody { + model?: string; + input?: string | ResponsesInputItem[]; + instructions?: string; + messages?: Array<{ role?: string; content?: unknown }>; + stream?: boolean; + metadata?: Record; + reasoning?: Record; + thinking?: Record; + tools?: Array<{ type?: string; name?: string } & Record>; + tool_choice?: unknown; + max_output_tokens?: number; + previous_response_id?: string; +} + +export type ResponseSessionDefaults = Pick< + ResponsesRequestBody, + 'instructions' | 'metadata' | 'tools' | 'tool_choice' +>; + +export interface ResponseSession { + accessKeyId: string | null; + credentialFilename: string | null; + createdAt: number; + id: string; + model: string; + transcript: TranscriptMessage[]; + defaults: ResponseSessionDefaults; + upstreamProtocol?: 'chat' | 'responses'; +} + +export interface ChatResponseToolCall { + index?: number; + id?: string; + type?: string; + function?: { + arguments?: string; + name?: string; + }; +} + +export interface ChatResponseMessage { + content?: unknown; + tool_calls?: ChatResponseToolCall[]; + /** Reasoning the upstream produced alongside `content`. */ + reasoning_content?: string; + reasoning?: string; +} + +export interface ChatImagePart { + image_url: { url: string }; + type: 'image_url'; +} + +export interface ChatTextPart { + text: string; + type: 'text'; +} + +export type ChatContentPart = string | ChatTextPart | ChatImagePart; + +/** + * Transcript content. Images are kept as structured parts so they survive the + * Chat-shaped round trip through the transcript and reach the model as images + * instead of a JSON dump. + */ +export type TranscriptContent = string | ChatContentPart[]; + +export interface TranscriptMessage { + role: string; + content: TranscriptContent | null; + tool_calls?: Array<{ + id: string; + type: string; + function: { + name: string; + arguments: string; + }; + }>; + tool_call_id?: string; + /** + * Prior-turn reasoning recovered from a replayed reasoning item. + * + * Carried on the assistant message the reasoning belongs to rather than sent + * as its own message: the chat upstream has no standalone reasoning entry, + * and a reasoning-only message would be an empty turn. + */ + reasoning?: string; +} + +export interface StreamingToolCallState { + addedEmitted: boolean; + arguments: string; + canonicalKey: string; + callId: string; + name: string; + outputIndex: number; + outputItemId: string; + pendingArgumentDeltas: string[]; +} + +export interface StreamingMessageState { + outputIndex: number | null; + outputItemId: string; +} + +export interface ResponsesServerToolItem { + completed: Record; + inProgress: Record; + outputIndex: number; +} + +export interface ResponseSessionMetadata { + bytes: number; + createdAt: number; +} + +export type SupportedResponsesTool = NonNullable< + ResponsesRequestBody['tools'] +>[number]; diff --git a/lib/server/proxy/server-tool/args.ts b/lib/server/proxy/server-tool/args.ts new file mode 100644 index 0000000..209bdf8 --- /dev/null +++ b/lib/server/proxy/server-tool/args.ts @@ -0,0 +1,108 @@ +import { asRecord } from '../../shared/content'; +import type { WebFetchQuery } from '../../search/types'; + +/** + * Reads one string field out of a tool-call argument object. + * + * Backends expect a single string, but models emit `query`, `q`, + * `search_query`, or an Anthropic-style `{query: {q: ...}}` nested object, so + * any string-ish value is accepted rather than failing the call. + */ +export const extractStringArgument = ({ + keys, + rawArguments, + required, +}: { + keys: string[]; + rawArguments: string | undefined; + required: boolean; +}): string => { + if (!rawArguments) { + return ''; + } + + try { + const parsed = JSON.parse(rawArguments) as unknown; + + // Some clients send a bare JSON string rather than an object. + if (typeof parsed === 'string') { + return parsed.trim(); + } + + const record = asRecord(parsed); + + if (!record) { + return ''; + } + + for (const key of keys) { + const value = record[key]; + + if (typeof value === 'string' && value.trim()) { + return value.trim(); + } + + // Anthropic-style arguments nest the value one level deeper. + const nested = asRecord(value); + + if (nested) { + for (const nestedKey of keys) { + const nestedValue = nested[nestedKey]; + + if (typeof nestedValue === 'string' && nestedValue.trim()) { + return nestedValue.trim(); + } + } + } + } + + if (required) { + // Fall back to whichever field holds the first non-empty string, so an + // unexpected argument shape still yields a usable value. Only safe when + // every field means the same thing, which is true for a single-string + // search query but not for a fetch's url plus prompt. + const firstString = Object.values(record).find( + (value): value is string => + typeof value === 'string' && value.trim().length > 0, + ); + + return firstString?.trim() ?? ''; + } + + return ''; + } catch { + // Malformed JSON: treat the raw text as the value so the call still runs. + return rawArguments.trim(); + } +}; + +export const extractSearchQuery = (rawArguments: string | undefined): string => + extractStringArgument({ + keys: ['query', 'q', 'search_query', 'text'], + rawArguments, + required: true, + }); + +/** + * Builds the `web_fetch` arguments. + * + * A missing URL is reported to the model rather than thrown: the model sent the + * call, so telling it the argument was missing lets it retry correctly, whereas + * an exception would surface as an opaque tool failure. + */ +export const extractFetchQuery = ( + rawArguments: string | undefined, +): WebFetchQuery => { + const url = extractStringArgument({ + keys: ['url', 'uri', 'link'], + rawArguments, + required: false, + }); + const prompt = extractStringArgument({ + keys: ['prompt', 'question', 'goal'], + rawArguments, + required: false, + }); + + return { ...(prompt ? { prompt } : {}), url }; +}; diff --git a/lib/server/proxy/server-tool/classify.ts b/lib/server/proxy/server-tool/classify.ts new file mode 100644 index 0000000..8fce0ac --- /dev/null +++ b/lib/server/proxy/server-tool/classify.ts @@ -0,0 +1,243 @@ +import { asRecord } from '../../shared/content'; +import { + buildWebFetchToolDefinition, + buildWebSearchToolDefinition, + isMarkedServerTool, + normalizeToolName, + stripServerToolMarker, + WEB_FETCH_TOOL_NAME, + WEB_FETCH_TOOL_TYPE_PREFIX, + WEB_SEARCH_TOOL_NAME, + WEB_SEARCH_TOOL_TYPE_PREFIX, +} from '../../search/tool'; +import type { WebFetchProvider, WebSearchProvider } from '../../search/types'; +import type { ChatCompletionToolCall } from './types'; + +/** + * Recognises one server-tool declaration. + * + * Anthropic sends dated server-tool *types* (`web_search_20260209`, + * `web_fetch_20250910`), Responses sends `web_search_preview`, and a client may + * also declare a plain function tool with the bare name for its own purposes. + * All three shapes have to match, because the tool has to be swapped for a + * function upstream can actually call regardless of how it arrived. + * + * Returns two independent answers. `matches` says the declaration is one this + * proxy can serve; `serverDeclared` says it arrived as a provider-executed + * server tool rather than as the client's own function. The difference decides + * what happens when the tool cannot be executed: a server-tool declaration is + * dropped, because upstream has no idea what to do with it, whereas the + * client's own function is left exactly as sent — the client is the one that + * resolves it, and deleting it would silently remove a capability the client + * asked for. + */ +export const classifyServerTool = ( + tool: unknown, + name: string, + prefix: string, +): { matches: boolean; serverDeclared: boolean } => { + const record = asRecord(tool); + + if (!record) { + return { matches: false, serverDeclared: false }; + } + + const type = typeof record.type === 'string' ? record.type : ''; + + // A dedicated server-tool type (`web_search_20260209`, `web_fetch_20250910`, + // `web_search_preview`) is unambiguous: only a provider-executed tool is + // declared that way. The trailing date is part of the version, not the name, + // so the prefix is matched in canonical form — `WebFetch_20250910` arrives + // from upstream as readily as its snake_case spelling. + if (normalizeToolName(type).startsWith(normalizeToolName(prefix))) { + return { matches: true, serverDeclared: true }; + } + + const fn = asRecord(record.function); + const isBareName = + (typeof fn?.name === 'string' && + normalizeToolName(fn.name) === normalizeToolName(name)) || + (typeof record.name === 'string' && + normalizeToolName(record.name).startsWith(normalizeToolName(prefix))); + + // A Responses translation has already flattened the declaration into a plain + // function, so the type is gone by now; its marker is the only surviving + // evidence that the client asked for a provider-executed tool. + return { + matches: isBareName, + serverDeclared: isBareName && isMarkedServerTool(tool), + }; +}; + +export const isWebSearchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) + .matches; + +export const isWebFetchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) + .matches; + +export const isServerDeclaredSearchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) + .serverDeclared; + +export const isServerDeclaredFetchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) + .serverDeclared; + +/** + * Whether `toolCall` is a call the proxy is meant to execute. + * + * Matched in canonical form because the name comes back from the model, which + * is under no obligation to repeat the spelling it was given: upstream echoes + * `web_fetch` as `WebFetch` often enough to matter here. A miss is not a + * fallback to the client — the call leaves the loop as an unanswered + * client-owned tool, so the fetch silently never happens. + */ +export const isWebSearchToolCall = ( + toolCall: ChatCompletionToolCall, +): boolean => { + return ( + typeof toolCall.function?.name === 'string' && + normalizeToolName(toolCall.function.name) === + normalizeToolName(WEB_SEARCH_TOOL_NAME) + ); +}; + +export const isWebFetchToolCall = ( + toolCall: ChatCompletionToolCall, +): boolean => { + return ( + typeof toolCall.function?.name === 'string' && + normalizeToolName(toolCall.function.name) === + normalizeToolName(WEB_FETCH_TOOL_NAME) + ); +}; + +/** + * Swaps locally executed server-tool declarations for functions upstream can + * call. Passthrough tools keep their upstream representation. + * + * Returns `null` when no web tool is present. `executes` distinguishes a local + * backend from passthrough: the latter still strips the internal provenance + * marker, but never starts the server loop or buffers a stream. + */ +export const replaceServerTools = ({ + fetchEnabled, + fetchProvider, + searchEnabled, + searchPassthrough, + searchProvider, + tools, +}: { + fetchEnabled: boolean; + fetchProvider: WebFetchProvider | null; + searchEnabled: boolean; + searchPassthrough: boolean; + searchProvider: WebSearchProvider | null; + tools: unknown; +}): { + executes: boolean; + /** Canonical names the proxy took over, so call classification can tell its own calls from a client's. */ + ownedNames: Set; + tools: unknown[]; +} | null => { + if (!Array.isArray(tools) || !tools.length) { + return null; + } + + let matched = false; + let executes = false; + // Names the proxy is executing itself. A client may declare its own tool + // under the same name, and the loop must not answer those calls: matching + // the name is not enough to own it. + const ownedNames = new Set(); + + const rewritten = tools.flatMap((tool): unknown[] => { + if (isWebSearchTool(tool)) { + if (searchEnabled && searchProvider) { + matched = true; + executes = true; + ownedNames.add(normalizeToolName(WEB_SEARCH_TOOL_NAME)); + + return [{ type: 'function', function: buildWebSearchToolDefinition() }]; + } + + if (!isServerDeclaredSearchTool(tool)) { + return [tool]; + } + + matched = true; + return searchPassthrough ? [stripServerToolMarker(tool)] : []; + } + + if (isWebFetchTool(tool)) { + // A client-owned function of the same name wins over the backend, exactly + // as it does for search. The backend setting chooses who runs the *proxy's* + // tool; it is not a licence to take over a tool the client declared and + // resolves itself. Without this, a client that ships its own `web_fetch` + // loses it the moment a deployment picks a backend. + if (!isServerDeclaredFetchTool(tool)) { + return [tool]; + } + + if (fetchEnabled && fetchProvider) { + matched = true; + executes = true; + ownedNames.add(normalizeToolName(WEB_FETCH_TOOL_NAME)); + + return [{ type: 'function', function: buildWebFetchToolDefinition() }]; + } + + matched = true; + return [stripServerToolMarker(tool)]; + } + + // The marker is internal to this proxy, so it never reaches upstream. + return [stripServerToolMarker(tool)]; + }); + + return matched ? { executes, ownedNames, tools: rewritten } : null; +}; + +/** + * Whether the proxy is the one meant to answer this call. + * + * A call without an available backend is not a fallback to the client — it + * leaves the loop as an unanswered client-owned tool — but it must not be + * counted as locally executable either. + */ +export const isLocalServerToolCall = ({ + fetchProvider, + ownedNames, + toolCall, + searchProvider, +}: { + fetchProvider: WebFetchProvider | null; + /** + * Canonical names the proxy took over. Without it a client's own tool that + * happens to share a name — `web_fetch`, which is not a server tool in the + * Responses API — gets executed by the loop instead of handed back. + */ + ownedNames?: Set; + toolCall: ChatCompletionToolCall; + searchProvider: WebSearchProvider | null; +}): boolean => + (Boolean(searchProvider) && + isWebSearchToolCall(toolCall) && + isOwned(ownedNames, WEB_SEARCH_TOOL_NAME)) || + (Boolean(fetchProvider) && + isWebFetchToolCall(toolCall) && + isOwned(ownedNames, WEB_FETCH_TOOL_NAME)); + +/** + * Whether the proxy owns calls to `name`. + * + * `undefined` means the caller predates ownership tracking; those callers only + * ever run the proxy's own declarations, so they are unaffected by client tools + * of the same name. + */ +export const isOwned = ( + ownedNames: Set | undefined, + name: string, +): boolean => !ownedNames || ownedNames.has(normalizeToolName(name)); diff --git a/lib/server/proxy/server-tool/execution.ts b/lib/server/proxy/server-tool/execution.ts new file mode 100644 index 0000000..839d4c8 --- /dev/null +++ b/lib/server/proxy/server-tool/execution.ts @@ -0,0 +1,146 @@ +import { runWebFetchResult, runWebSearchResult } from '../../search'; +import type { WebFetchProvider, WebSearchProvider } from '../../search/types'; +import { asRecord } from '../../shared/content'; +import { extractFetchQuery, extractSearchQuery } from './args'; +import { isWebFetchToolCall } from './classify'; +import { + SERVER_TOOL_STREAM_EVENT_KEY, + type ChatCompletionToolCall, + type ServerToolCallbacks, + type ServerToolExecution, + type ServerToolInvocation, + type ServerToolStreamEvent, + type ServerToolTurn, +} from './types'; + +export const getServerToolStreamEvent = ( + value: unknown, +): ServerToolStreamEvent | null => { + const record = asRecord(value); + const event = asRecord(record?.[SERVER_TOOL_STREAM_EVENT_KEY]); + + if (event?.phase === 'call' && event.invocation) { + return { + invocation: event.invocation as ServerToolInvocation, + phase: 'call', + }; + } + + if (event?.phase === 'result' && event.execution) { + return { + execution: event.execution as ServerToolExecution, + phase: 'result', + }; + } + + return null; +}; + +const serverToolExecutions = new WeakMap(); + +export const attachServerToolExecutions = ( + response: Response, + executions: ServerToolExecution[], +): Response => { + if (executions.length) { + serverToolExecutions.set(response, executions); + } + + return response; +}; + +export const getServerToolExecutions = ( + response: Response, +): ServerToolExecution[] => serverToolExecutions.get(response) ?? []; + +/** + * Per-hop grouping, kept off the wire for the same reason `executions` is: it + * is not part of the OpenAI protocol, so a `/v1/chat/completions` client must + * not see it — a strict validator can reject the extra field, and the tool + * data would otherwise be sent twice. + */ +const serverToolTurns = new WeakMap(); + +export const attachServerToolTurns = ( + response: Response, + turns: ServerToolTurn[], +): Response => { + if (turns.length) { + serverToolTurns.set(response, turns); + } + + return response; +}; + +export const getServerToolTurns = ( + response: Response, +): ServerToolTurn[] | undefined => serverToolTurns.get(response); + +export const buildServerToolInvocation = ( + toolCall: ChatCompletionToolCall, + iteration: number, + index: number, +): ServerToolInvocation => + isWebFetchToolCall(toolCall) + ? { + id: toolCall.id ?? `server_tool_${iteration}_${index}`, + input: extractFetchQuery(toolCall.function?.arguments), + type: 'web_fetch', + } + : { + id: toolCall.id ?? `server_tool_${iteration}_${index}`, + input: { query: extractSearchQuery(toolCall.function?.arguments) }, + type: 'web_search', + }; + +export const executeServerToolInvocations = async ({ + callbacks, + fetchProvider, + invocations, + searchProvider, +}: { + callbacks?: ServerToolCallbacks; + fetchProvider: WebFetchProvider | null; + invocations: ServerToolInvocation[]; + searchProvider: WebSearchProvider | null; +}): Promise< + Array<{ + content: string; + execution: ServerToolExecution; + tool_call_id: string; + }> +> => { + invocations.forEach((invocation) => callbacks?.onCall?.(invocation)); + + return await Promise.all( + invocations.map(async (invocation) => { + if (invocation.type === 'web_fetch') { + const result = await runWebFetchResult({ + provider: fetchProvider, + query: invocation.input, + }); + const execution: ServerToolExecution = { ...invocation, result }; + callbacks?.onResult?.(execution); + + return { + content: result.content, + execution, + tool_call_id: invocation.id, + }; + } + + const result = await runWebSearchResult({ + provider: searchProvider, + query: invocation.input.query, + }); + const execution: ServerToolExecution = { ...invocation, result }; + callbacks?.onResult?.(execution); + + return { + content: result.content, + execution, + tool_call_id: invocation.id, + }; + }), + ); +}; diff --git a/lib/server/proxy/server-tool/payload.ts b/lib/server/proxy/server-tool/payload.ts new file mode 100644 index 0000000..1339645 --- /dev/null +++ b/lib/server/proxy/server-tool/payload.ts @@ -0,0 +1,98 @@ +import { extractErrorMessage } from '../../shared/http'; +import type { ChatCompletionPayload } from './types'; + +/** + * Rebuilds a failed upstream response so its body can be read again. + * + * A `Response` body can only be consumed once. The loop reads it to decide + * whether the model asked for a server tool, and handing the same object back + * used to leave the route layer — which reads it again to build the answer the + * client actually sees — with a spent body: the second read threw + * "Body already used" and the client got a 500 in place of the real upstream + * status. Draining it here and replaying the bytes in a fresh response keeps + * both reads working and preserves the body verbatim, so an upstream error + * detail that is not valid JSON still reaches the client intact. + * + * `content-length` and `content-encoding` are dropped: the body is re-emitted + * rather than re-encoded, and a stale length would describe bytes the upstream + * compressed before this layer ever saw them. + */ +export const buildServerToolFailureResponse = async ( + response: Response, +): Promise => { + const headers = new Headers(response.headers); + + headers.delete('content-length'); + headers.delete('content-encoding'); + headers.set('content-type', 'application/json'); + + return new Response(await response.text(), { + headers, + status: response.status, + statusText: response.statusText, + }); +}; + +/** + * Parses a buffered upstream body, tolerating a failure that is not JSON. + * + * A successful response must be well-formed — anything else is a bug worth + * surfacing — but a failure status already tells the caller everything it + * needs to know, and its body may legitimately be an HTML error page or a + * bare string. Rejecting on those would turn an ordinary outage into an + * unhandled rejection. + */ +export const parseBufferedPayload = ( + buffered: string, + ok: boolean, +): ChatCompletionPayload => { + try { + return JSON.parse(buffered) as ChatCompletionPayload; + } catch (error) { + if (ok) { + throw error; + } + + return {}; + } +}; + +export const readBufferedChatCompletionPayload = async ( + response: Response, +): Promise => { + // Cloned so the failure path can replay the body verbatim; see + // {@link buildServerToolFailureResponse}. + const buffered = await response.clone().text(); + const payload = parseBufferedPayload(buffered, response.ok); + + if (!response.ok || payload.error) { + const ownMessage = payload.error?.message; + // `extractErrorMessage` digs a nested message out of the payload, so + // `{"error":{"message":"x"}}` reaches the client as "x" rather than as a + // JSON string. The raw body is the fallback: a payload carrying only a + // code has no message to find, and the JSON is still the only record of + // what happened. An empty body says nothing, so it falls all the way + // through to the generic message instead of winning on being non-null. + const detail = buffered.trim(); + + return { + ...payload, + error: { + // The upstream's own explanation — a rate-limit code, a reset + // timestamp — beats the proxy's generic "Upstream CodeBuddy request + // failed", which says only that something failed and leaves the client + // no way to tell what. + // + // `status` travels with the frame so a downstream mapper can name the + // real error type instead of guessing it from the message text. + message: + extractErrorMessage(payload) ?? + ownMessage ?? + (detail || `Upstream request failed with status ${response.status}`), + ...(response.ok ? {} : { status: response.status }), + }, + }; + } + + return payload; +}; diff --git a/lib/server/proxy/server-tool/sse.ts b/lib/server/proxy/server-tool/sse.ts new file mode 100644 index 0000000..7a87c86 --- /dev/null +++ b/lib/server/proxy/server-tool/sse.ts @@ -0,0 +1,130 @@ +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; +import { + STREAM_TEXT_CHUNK_LENGTH, + type ChatCompletionPayload, + type JsonRecord, +} from './types'; + +/** + * Replays a buffered completion as chat-completion SSE. Used only after a + * streaming request actually invokes a server-executed tool; ordinary answers + * keep the upstream response untouched. + */ +export const synthesizeChatCompletionStream = ( + payload: ChatCompletionPayload, + fallbackModel: string, +): Response => { + const encoder = new TextEncoder(); + const choice = payload.choices?.[0]; + const message = choice?.message; + const created = + typeof payload.created === 'number' + ? payload.created + : Math.floor(Date.now() / 1000); + const model = + typeof payload.model === 'string' && payload.model + ? payload.model + : fallbackModel; + const id = + typeof payload.id === 'string' && payload.id + ? payload.id + : `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`; + + const enqueue = ( + controller: ReadableStreamDefaultController, + chunk: JsonRecord, + ): void => { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + ...chunk, + created, + id, + model, + object: 'chat.completion.chunk', + })}\n\n`, + ), + ); + }; + + const stream = new ReadableStream({ + start: (controller) => { + enqueue(controller, { + choices: [{ delta: { role: 'assistant' }, index: 0 }], + }); + + const reasoning = message?.reasoning_content ?? message?.reasoning; + + if (reasoning) { + enqueue(controller, { + choices: [{ delta: { reasoning_content: reasoning }, index: 0 }], + }); + } + + const content = + typeof message?.content === 'string' ? message.content : ''; + + for ( + let offset = 0; + offset < content.length; + offset += STREAM_TEXT_CHUNK_LENGTH + ) { + enqueue(controller, { + choices: [ + { + delta: { + content: content.slice( + offset, + offset + STREAM_TEXT_CHUNK_LENGTH, + ), + }, + index: 0, + }, + ], + }); + } + + const passthroughCalls = (message?.tool_calls ?? []).map( + (toolCall, index) => ({ + ...toolCall, + id: toolCall.id ?? `call_${index}`, + index, + type: toolCall.type ?? 'function', + }), + ); + + if (passthroughCalls.length) { + enqueue(controller, { + choices: [{ delta: { tool_calls: passthroughCalls }, index: 0 }], + }); + } + + enqueue(controller, { + choices: [ + { + delta: {}, + finish_reason: + choice?.finish_reason ?? + (passthroughCalls.length ? 'tool_calls' : 'stop'), + index: 0, + }, + ], + }); + + if (payload.usage !== undefined && payload.usage !== null) { + enqueue(controller, { + choices: [], + usage: payload.usage, + }); + } + + controller.enqueue(encodeDoneFrame()); + controller.close(); + }, + }); + + return createSseResponse(stream, { + headers: { 'Access-Control-Allow-Origin': '*' }, + status: 200, + }); +}; diff --git a/lib/server/proxy/server-tool/stream.ts b/lib/server/proxy/server-tool/stream.ts new file mode 100644 index 0000000..8233816 --- /dev/null +++ b/lib/server/proxy/server-tool/stream.ts @@ -0,0 +1,230 @@ +import { isLocalServerToolCall } from './classify'; +import type { ChatCompletionMessage, ChatCompletionToolCall } from './types'; +import type { WebFetchProvider, WebSearchProvider } from '../../search/types'; + +export const mergeStreamingToolName = ( + previous: string, + incoming: string, +): string => { + if (!previous || incoming.startsWith(previous)) return incoming; + if (!incoming || previous.endsWith(incoming)) return previous; + return previous + incoming; +}; + +export const aggregateStreamingToolCalls = ( + deltas: ChatCompletionToolCall[], +): ChatCompletionToolCall[] => { + const calls = new Map< + string, + ChatCompletionToolCall & { + function: { arguments: string; name: string }; + } + >(); + const latestKeyByIndex = new Map(); + + deltas.forEach((delta, position) => { + const indexedKey = + typeof delta.index === 'number' + ? latestKeyByIndex.get(delta.index) + : undefined; + const key = + indexedKey ?? + (delta.id ? `id:${delta.id}` : undefined) ?? + (typeof delta.index === 'number' + ? `index:${delta.index}` + : `position:${position}`); + const current = calls.get(key) ?? { + function: { arguments: '', name: '' }, + index: delta.index, + }; + + current.id = delta.id ?? current.id; + current.index = delta.index ?? current.index; + current.type = delta.type ?? current.type; + current.function.arguments += delta.function?.arguments ?? ''; + current.function.name = mergeStreamingToolName( + current.function.name, + delta.function?.name ?? '', + ); + calls.set(key, current); + + if (typeof delta.index === 'number') { + latestKeyByIndex.set(delta.index, key); + } + }); + + return [...calls.values()]; +}; + +/** + * Result of streaming one upstream response while watching for server-tool + * calls, so a follow-up iteration can decide what happened. + * + * `localCalls` and `remainingCalls` partition the aggregated tool calls the + * way the execution loop needs them; `frames` are the frames that were held + * back because they carried tool-call deltas. + */ +export interface ServerToolProbe { + content: string; + frames: string[]; + localCalls: ChatCompletionToolCall[]; + reasoning: string; + remainingCalls: ChatCompletionToolCall[]; + role: string; + toolCalls: ChatCompletionToolCall[]; + usage: unknown; +} + +export const probeServerToolStream = async ({ + canContinue, + context, + emitRaw, + fetchProvider, + onReader, + ownedNames, + response, + searchProvider, +}: { + canContinue: () => boolean; + context: { + responseCreated: number; + responseId: string; + responseModel: string; + responseObject: string; + role: string; + usage: unknown; + }; + emitRaw: (frame: string) => void; + fetchProvider: WebFetchProvider | null; + ownedNames?: Set; + /** + * Hands the active reader to the caller's cancellation path. Without it a + * disconnect cannot interrupt a read that is already parked: the loop only + * notices the cancellation once upstream produces another chunk, which a + * stalled upstream never does. + */ + onReader?: (reader: ReadableStreamDefaultReader | null) => void; + response: Response; + searchProvider: WebSearchProvider | null; +}): Promise => { + const frames: string[] = []; + const toolCallDeltas: ChatCompletionToolCall[] = []; + const decoder = new TextDecoder(); + const reader = response.body!.getReader(); + onReader?.(reader); + let buffer = ''; + let content = ''; + let reasoning = ''; + + const inspectFrame = (frame: string): void => { + const line = frame + .split(/\r?\n/) + .find((segment) => segment.startsWith('data:')); + + if (!line) { + emitRaw(frame); + return; + } + + const raw = line.slice(5).trim(); + if (!raw) return; + if (raw === '[DONE]') { + frames.push(frame); + return; + } + + try { + const chunk = JSON.parse(raw) as { + choices?: Array<{ + delta?: ChatCompletionMessage & { + tool_calls?: ChatCompletionToolCall[]; + }; + finish_reason?: string | null; + }>; + created?: number; + id?: string; + model?: string; + object?: string; + usage?: unknown; + }; + context.responseId = chunk.id ?? context.responseId; + context.responseModel = chunk.model ?? context.responseModel; + context.responseObject = + chunk.object?.replace(/\.chunk$/, '') ?? context.responseObject; + context.responseCreated = chunk.created ?? context.responseCreated; + context.usage = chunk.usage ?? context.usage; + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + context.role = delta?.role ?? context.role; + content += delta?.content ?? ''; + reasoning += delta?.reasoning_content ?? delta?.reasoning ?? ''; + + // A tool-call frame is held rather than forwarded: if the turn turns out + // to invoke a server tool, the call has to be answered locally instead + // of being handed to the client as an unresolved call. Anything else the + // delta carried — most importantly the text the model wrote before + // deciding to search — still belongs to the visible turn, so it is + // re-emitted without the tool call. + if (delta?.tool_calls?.length) { + toolCallDeltas.push(...delta.tool_calls); + frames.push(frame); + + const visibleDelta = { ...delta }; + delete visibleDelta.tool_calls; + + if (Object.keys(visibleDelta).length) { + const visible = JSON.stringify({ + ...chunk, + choices: [{ ...choice, delta: visibleDelta, finish_reason: null }], + }); + emitRaw(`data: ${visible}`); + } + return; + } + + if (choice?.finish_reason === 'tool_calls') { + frames.push(frame); + return; + } + } catch { + emitRaw(frame); + return; + } + + emitRaw(frame); + }; + + while (true) { + const chunk = await reader.read(); + if (!canContinue()) break; + if (chunk.done) break; + buffer += decoder.decode(chunk.value, { stream: true }); + const split = buffer.split(/\r?\n\r?\n/); + buffer = split.pop() ?? ''; + split.forEach(inspectFrame); + } + + if (buffer.trim()) inspectFrame(buffer); + reader.releaseLock(); + onReader?.(null); + + const toolCalls = aggregateStreamingToolCalls(toolCallDeltas); + const isLocalCall = (toolCall: ChatCompletionToolCall): boolean => + isLocalServerToolCall({ + fetchProvider, + ownedNames, + searchProvider, + toolCall, + }); + + return { + content, + frames, + localCalls: toolCalls.filter(isLocalCall), + reasoning, + remainingCalls: toolCalls.filter((toolCall) => !isLocalCall(toolCall)), + role: context.role, + toolCalls, + usage: context.usage, + }; +}; diff --git a/lib/server/proxy/server-tool/turns.ts b/lib/server/proxy/server-tool/turns.ts new file mode 100644 index 0000000..877b9e2 --- /dev/null +++ b/lib/server/proxy/server-tool/turns.ts @@ -0,0 +1,205 @@ +import { asRecord, readReasoning } from '../../shared/content'; +import type { + ChatCompletionMessage, + ChatCompletionPayload, + ChatCompletionToolCall, + JsonRecord, + ServerToolExecution, + ServerToolTurn, +} from './types'; + +export const sumUsage = (accumulated: unknown, incoming: unknown): unknown => { + const left = asRecord(accumulated); + const right = asRecord(incoming); + + if (!left) { + return incoming ?? null; + } + + if (!right) { + return accumulated; + } + + const merged: JsonRecord = { ...left }; + + for (const [key, value] of Object.entries(right)) { + const previous = left[key]; + + if (typeof value === 'number' && typeof previous === 'number') { + merged[key] = previous + value; + } else if (value !== undefined) { + merged[key] = value; + } + } + + return merged; +}; + +/** + * Folds completed search results into the assistant text and re-emits the + * outstanding tool calls unchanged, so a turn that mixed search with + * client-side calls stays a valid transcript. The client sees its own calls + * come back as if upstream had returned them directly. + * + * The findings are folded only when the route has no other way to carry them. + * A route that renders them structurally passes + * `findingsAsStructuredBlocks`, and the text is left alone: the results are + * already on the wire as a result block, and a second copy in the prose is + * what the user reads as the model reciting its own search output. + */ +export const buildMixedTurnPayload = ({ + findingsAsStructuredBlocks = false, + message, + payload, + remainingCalls, + searchResults, + usage, +}: { + findingsAsStructuredBlocks?: boolean; + message: ChatCompletionMessage | undefined; + payload: ChatCompletionPayload; + remainingCalls: ChatCompletionToolCall[]; + searchResults: string[]; + usage?: unknown; +}): ChatCompletionPayload => { + const existingText = + typeof message?.content === 'string' && message.content.trim() + ? message.content.trim() + : ''; + const findings = findingsAsStructuredBlocks + ? '' + : searchResults.filter(Boolean).join('\n\n'); + const content = [existingText, findings].filter(Boolean).join('\n\n'); + + return { + ...payload, + ...(usage ? { usage } : {}), + choices: (payload.choices ?? []).map((choice, index) => + index === 0 + ? { + ...choice, + finish_reason: 'tool_calls', + message: { + ...(choice.message ?? {}), + content: content || null, + role: 'assistant', + tool_calls: remainingCalls, + }, + } + : choice, + ), + }; +}; + +/** + * Pairs each hop's reasoning and text with the calls that hop made. + * + * The three arrays are index-aligned — entry N is hop N — so zipping them back + * together is what restores the grouping a joined string cannot express. `texts` + * and `reasonings` are one entry longer than `executions`, because the closing + * hop answers instead of calling another tool. + */ +export const buildIntermediateTurns = ({ + executions, + reasonings, + texts, +}: { + executions: ServerToolExecution[][]; + reasonings: string[]; + texts: string[]; +}): ServerToolTurn[] => + Array.from( + { length: Math.max(reasonings.length, texts.length) }, + (_, index) => ({ + // Only `executions` can run short: the closing hop answers without + // calling anything, so it has an entry in the prose arrays but none here. + // The three arrays stay aligned because every hop appends to all of them. + executions: executions[index] ?? [], + reasoning: reasonings[index], + text: texts[index], + }), + ); + +/** + * Folds the text a multi-hop turn produced before its later server-tool calls + * into the payload the client receives. + * + * Only the last iteration's message is in `payload`, but a turn that searched + * more than once spoke before each search, and that text is part of the turn: + * dropping it hides the model's reasoning from the user and leaves the + * client's transcript out of step with what the model actually said. + * + * The same hops are also re-grouped into the `turns` half of the result, + * because the folded strings cannot express where one hop ends and the next + * begins. That half travels beside the response rather than inside it: the + * payload is what an OpenAI-protocol client receives, and `turns` is not part + * of that protocol, so it is handed over out of band like `executions`. + * + * Shared with the image-generation loop, which has the same shape: a local + * tool call is replayed with its result appended, so only the final hop's + * message would otherwise survive. + */ +export const withIntermediateTurns = ({ + executions, + payload, + reasonings, + texts, +}: { + executions: ServerToolExecution[][]; + payload: ChatCompletionPayload; + reasonings: string[]; + texts: string[]; +}): { payload: ChatCompletionPayload; turns: ServerToolTurn[] } => { + const extraText = texts.filter(Boolean).join('\n\n'); + const extraReasoning = reasonings.filter(Boolean).join('\n\n'); + const [first, ...rest] = payload.choices ?? []; + + if (!first) { + return { payload, turns: [] }; + } + + const message = first.message ?? {}; + const existingText = + typeof message.content === 'string' ? message.content : ''; + + // Nothing from the earlier hops and nothing to fold in — but the hops may + // still have run tools, which is exactly the case a caller consuming `turns` + // needs: a hop that called a tool without speaking first is still a hop. + if (!extraText && !extraReasoning && !executions.length) { + return { payload, turns: [] }; + } + + const content = [extraText, existingText].filter(Boolean).join('\n\n'); + const reasoning = [extraReasoning, readReasoning(message)] + .filter(Boolean) + .join('\n\n'); + + return { + payload: { + ...payload, + choices: [ + { + ...first, + message: { + ...message, + content, + ...(reasoning ? { reasoning_content: reasoning } : {}), + }, + }, + ...rest, + ], + }, + // Per-hop grouping for renderers that can express it. The joined strings + // above stay as the OpenAI-shaped view; a client that builds Anthropic + // content blocks needs to know where one hop's reasoning ends and the next + // begins, which a joined string has already lost. The closing hop is the + // model's final answer, so it carries no further calls. + turns: buildIntermediateTurns({ + executions, + reasonings: [...reasonings, readReasoning(message)], + texts: [...texts, existingText], + }), + }; +}; + +export { readReasoning }; diff --git a/lib/server/proxy/server-tool/types.ts b/lib/server/proxy/server-tool/types.ts new file mode 100644 index 0000000..89c5c97 --- /dev/null +++ b/lib/server/proxy/server-tool/types.ts @@ -0,0 +1,129 @@ +import type { + WebFetchQuery, + WebFetchResponse, + WebSearchResponse, +} from '../../search/types'; +import type { ChatRequestBody } from '../codebuddy'; + +export const MAX_SEARCH_ITERATIONS = 5; +export const STREAM_TEXT_CHUNK_LENGTH = 1024; + +export type JsonRecord = Record; + +export interface ChatCompletionToolCall { + id?: string; + index?: number; + type?: string; + function?: { + arguments?: string; + name?: string; + }; +} + +export interface ChatCompletionMessage { + content?: string | null; + reasoning?: string; + reasoning_content?: string; + role?: string; + tool_calls?: ChatCompletionToolCall[]; +} + +export interface ChatCompletionPayload { + choices?: Array<{ + finish_reason?: string | null; + index?: number; + message?: ChatCompletionMessage; + }>; + created?: number; + /** + * `status` is the upstream HTTP status, carried so a downstream mapper can + * name the real error type instead of guessing it from the message text. It + * is absent for a payload that already reported an error of its own. + */ + error?: { message?: string; status?: number }; + id?: string; + model?: string; + object?: string; + usage?: unknown; +} + +/** + * Result of one server-tool pass. + * + * `response` is null when no tool could be executed: the request still has to + * be sent, but with the server-tool declarations already stripped, so the + * caller falls through to its ordinary upstream path. + */ +export interface ServerToolLoopResult { + body: ChatRequestBody; + executions: ServerToolExecution[]; + response: Response | null; + /** + * One entry per server-tool hop, in the order the model produced them. + * + * Carries the grouping `message.content` / `reasoning_content` cannot: a + * multi-hop turn joins every hop into one string per kind, which loses where + * one hop's reasoning ends and the next begins. Travels beside the response + * rather than inside it, because it is not part of the OpenAI protocol — a + * block renderer reads it off the response through `getServerToolTurns`. + * Empty when no hop ran. + */ + turns: ServerToolTurn[]; +} + +export type ServerToolInvocation = + | { + id: string; + input: { query: string }; + type: 'web_search'; + } + | { + id: string; + input: WebFetchQuery; + type: 'web_fetch'; + }; + +export type ServerToolExecution = + | (Extract & { + result: WebSearchResponse; + }) + | (Extract & { + result: WebFetchResponse; + }); + +/** + * One server-tool hop as the model produced it. + * + * `reasoning` and `text` are what the model wrote before the calls in + * `executions`; both are empty when it called tools without speaking first. The + * last hop of a turn usually has no executions, because the model answered + * instead of reaching for another tool. + */ +export interface ServerToolTurn { + executions: ServerToolExecution[]; + reasoning: string; + text: string; +} + +export interface ServerToolCallbacks { + emitStreamEvents?: boolean; + /** + * Set by routes that render a server tool's findings structurally — + * Anthropic's `web_search_tool_result` block — instead of as prose. Those + * routes must not also fold the same findings into the assistant text, or + * the user sees the results twice: once as a result block and once as if + * the model had written them. + */ + findingsAsStructuredBlocks?: boolean; + onCall?: (invocation: ServerToolInvocation) => void; + onResult?: (execution: ServerToolExecution) => void; +} + +export const SERVER_TOOL_STREAM_EVENT_KEY = 'x-codebuddy2api-server-tool'; + +export type ServerToolStreamEvent = + | { invocation: ServerToolInvocation; phase: 'call' } + | { execution: ServerToolExecution; phase: 'result' }; + +export type ServerToolUpstreamMode = + 'buffer' | 'detect-both' | 'detect-fetch' | 'detect-search' | 'stream'; diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index 55d4518..92ee8a6 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -4,34 +4,52 @@ import { isWebFetchEnabled, isWebSearchEnabled, } from '../domain/config'; -import { - resolveFetchProvider, - resolveSearchProvider, - runWebFetchResult, - runWebSearchResult, -} from '../search'; -import { extractErrorMessage } from '../shared/http'; +import { resolveFetchProvider, resolveSearchProvider } from '../search'; +import { normalizeSearchBackend } from '../search/tool'; +import type { WebFetchProvider, WebSearchProvider } from '../search/types'; +import { encodeDoneFrame } from '../shared/sse'; import type { ChatRequestBody } from './codebuddy'; import { - buildWebFetchToolDefinition, - buildWebSearchToolDefinition, - isMarkedServerTool, - normalizeSearchBackend, - normalizeToolName, - stripServerToolMarker, - WEB_FETCH_TOOL_NAME, - WEB_FETCH_TOOL_TYPE_PREFIX, - WEB_SEARCH_TOOL_NAME, - WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; -import type { - WebFetchProvider, - WebFetchQuery, - WebFetchResponse, - WebSearchProvider, - WebSearchResponse, -} from '../search/types'; + isLocalServerToolCall, + isWebFetchTool, + isWebSearchTool, + replaceServerTools, +} from './server-tool/classify'; +import { + buildServerToolInvocation, + executeServerToolInvocations, +} from './server-tool/execution'; +import { + buildServerToolFailureResponse, + parseBufferedPayload, + readBufferedChatCompletionPayload, +} from './server-tool/payload'; +import { synthesizeChatCompletionStream } from './server-tool/sse'; +import { + type ServerToolProbe, + probeServerToolStream, +} from './server-tool/stream'; +import { + buildMixedTurnPayload, + readReasoning, + sumUsage, + withIntermediateTurns, +} from './server-tool/turns'; +import { + MAX_SEARCH_ITERATIONS, + SERVER_TOOL_STREAM_EVENT_KEY, + type ChatCompletionMessage, + type ChatCompletionPayload, + type ChatCompletionToolCall, + type JsonRecord, + type ServerToolCallbacks, + type ServerToolExecution, + type ServerToolLoopResult, + type ServerToolStreamEvent, + type ServerToolTurn, + type ServerToolUpstreamMode, +} from './server-tool/types'; /** * Server-side web search for upstreams that do not implement it. @@ -49,1122 +67,6 @@ import type { * is buffered because its arguments are only complete once that response ends. */ -const MAX_SEARCH_ITERATIONS = 5; -const STREAM_TEXT_CHUNK_LENGTH = 1024; - -type JsonRecord = Record; - -export interface ChatCompletionToolCall { - id?: string; - index?: number; - type?: string; - function?: { - arguments?: string; - name?: string; - }; -} - -export interface ChatCompletionMessage { - content?: string | null; - reasoning?: string; - reasoning_content?: string; - role?: string; - tool_calls?: ChatCompletionToolCall[]; -} - -export interface ChatCompletionPayload { - choices?: Array<{ - finish_reason?: string | null; - index?: number; - message?: ChatCompletionMessage; - }>; - created?: number; - /** - * `status` is the upstream HTTP status, carried so a downstream mapper can - * name the real error type instead of guessing it from the message text. It - * is absent for a payload that already reported an error of its own. - */ - error?: { message?: string; status?: number }; - id?: string; - model?: string; - object?: string; - usage?: unknown; -} - -/** - * Rebuilds a failed upstream response so its body can be read again. - * - * A `Response` body can only be consumed once. The loop reads it to decide - * whether the model asked for a server tool, and handing the same object back - * used to leave the route layer — which reads it again to build the answer the - * client actually sees — with a spent body: the second read threw - * "Body already used" and the client got a 500 in place of the real upstream - * status. Draining it here and replaying the bytes in a fresh response keeps - * both reads working and preserves the body verbatim, so an upstream error - * detail that is not valid JSON still reaches the client intact. - * - * `content-length` and `content-encoding` are dropped: the body is re-emitted - * rather than re-encoded, and a stale length would describe bytes the upstream - * compressed before this layer ever saw them. - */ -const buildServerToolFailureResponse = async ( - response: Response, -): Promise => { - const headers = new Headers(response.headers); - - headers.delete('content-length'); - headers.delete('content-encoding'); - headers.set('content-type', 'application/json'); - - return new Response(await response.text(), { - headers, - status: response.status, - statusText: response.statusText, - }); -}; - -/** - * Parses a buffered upstream body, tolerating a failure that is not JSON. - * - * A successful response must be well-formed — anything else is a bug worth - * surfacing — but a failure status already tells the caller everything it - * needs to know, and its body may legitimately be an HTML error page or a - * bare string. Rejecting on those would turn an ordinary outage into an - * unhandled rejection. - */ -const parseBufferedPayload = ( - buffered: string, - ok: boolean, -): ChatCompletionPayload => { - try { - return JSON.parse(buffered) as ChatCompletionPayload; - } catch (error) { - if (ok) { - throw error; - } - - return {}; - } -}; - -const readBufferedChatCompletionPayload = async ( - response: Response, -): Promise => { - // Cloned so the failure path can replay the body verbatim; see - // {@link buildServerToolFailureResponse}. - const buffered = await response.clone().text(); - const payload = parseBufferedPayload(buffered, response.ok); - - if (!response.ok || payload.error) { - const ownMessage = payload.error?.message; - // `extractErrorMessage` digs a nested message out of the payload, so - // `{"error":{"message":"x"}}` reaches the client as "x" rather than as a - // JSON string. The raw body is the fallback: a payload carrying only a - // code has no message to find, and the JSON is still the only record of - // what happened. An empty body says nothing, so it falls all the way - // through to the generic message instead of winning on being non-null. - const detail = buffered.trim(); - - return { - ...payload, - error: { - // The upstream's own explanation — a rate-limit code, a reset - // timestamp — beats the proxy's generic "Upstream CodeBuddy request - // failed", which says only that something failed and leaves the client - // no way to tell what. - // - // `status` travels with the frame so a downstream mapper can name the - // real error type instead of guessing it from the message text. - message: - extractErrorMessage(payload) ?? - ownMessage ?? - (detail || `Upstream request failed with status ${response.status}`), - ...(response.ok ? {} : { status: response.status }), - }, - }; - } - - return payload; -}; - -const asRecord = (value: unknown): JsonRecord | null => { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as JsonRecord) - : null; -}; - -/** - * Recognises one server-tool declaration. - * - * Anthropic sends dated server-tool *types* (`web_search_20260209`, - * `web_fetch_20250910`), Responses sends `web_search_preview`, and a client may - * also declare a plain function tool with the bare name for its own purposes. - * All three shapes have to match, because the tool has to be swapped for a - * function upstream can actually call regardless of how it arrived. - * - * Returns two independent answers. `matches` says the declaration is one this - * proxy can serve; `serverDeclared` says it arrived as a provider-executed - * server tool rather than as the client's own function. The difference decides - * what happens when the tool cannot be executed: a server-tool declaration is - * dropped, because upstream has no idea what to do with it, whereas the - * client's own function is left exactly as sent — the client is the one that - * resolves it, and deleting it would silently remove a capability the client - * asked for. - */ -const classifyServerTool = ( - tool: unknown, - name: string, - prefix: string, -): { matches: boolean; serverDeclared: boolean } => { - const record = asRecord(tool); - - if (!record) { - return { matches: false, serverDeclared: false }; - } - - const type = typeof record.type === 'string' ? record.type : ''; - - // A dedicated server-tool type (`web_search_20260209`, `web_fetch_20250910`, - // `web_search_preview`) is unambiguous: only a provider-executed tool is - // declared that way. The trailing date is part of the version, not the name, - // so the prefix is matched in canonical form — `WebFetch_20250910` arrives - // from upstream as readily as its snake_case spelling. - if (normalizeToolName(type).startsWith(normalizeToolName(prefix))) { - return { matches: true, serverDeclared: true }; - } - - const fn = asRecord(record.function); - const isBareName = - (typeof fn?.name === 'string' && - normalizeToolName(fn.name) === normalizeToolName(name)) || - (typeof record.name === 'string' && - normalizeToolName(record.name).startsWith(normalizeToolName(prefix))); - - // A Responses translation has already flattened the declaration into a plain - // function, so the type is gone by now; its marker is the only surviving - // evidence that the client asked for a provider-executed tool. - return { - matches: isBareName, - serverDeclared: isBareName && isMarkedServerTool(tool), - }; -}; - -const isWebSearchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) - .matches; - -const isWebFetchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) - .matches; - -const isServerDeclaredSearchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) - .serverDeclared; - -const isServerDeclaredFetchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) - .serverDeclared; - -/** - * Whether `toolCall` is a call the proxy is meant to execute. - * - * Matched in canonical form because the name comes back from the model, which - * is under no obligation to repeat the spelling it was given: upstream echoes - * `web_fetch` as `WebFetch` often enough to matter here. A miss is not a - * fallback to the client — the call leaves the loop as an unanswered - * client-owned tool, so the fetch silently never happens. - */ -const isWebSearchToolCall = (toolCall: ChatCompletionToolCall): boolean => { - return ( - typeof toolCall.function?.name === 'string' && - normalizeToolName(toolCall.function.name) === - normalizeToolName(WEB_SEARCH_TOOL_NAME) - ); -}; - -const isWebFetchToolCall = (toolCall: ChatCompletionToolCall): boolean => { - return ( - typeof toolCall.function?.name === 'string' && - normalizeToolName(toolCall.function.name) === - normalizeToolName(WEB_FETCH_TOOL_NAME) - ); -}; - -/** - * Swaps locally executed server-tool declarations for functions upstream can - * call. Passthrough tools keep their upstream representation. - * - * Returns `null` when no web tool is present. `executes` distinguishes a local - * backend from passthrough: the latter still strips the internal provenance - * marker, but never starts the server loop or buffers a stream. - */ -const replaceServerTools = ({ - fetchEnabled, - fetchProvider, - searchEnabled, - searchPassthrough, - searchProvider, - tools, -}: { - fetchEnabled: boolean; - fetchProvider: WebFetchProvider | null; - searchEnabled: boolean; - searchPassthrough: boolean; - searchProvider: WebSearchProvider | null; - tools: unknown; -}): { - executes: boolean; - /** Canonical names the proxy took over, so call classification can tell its own calls from a client's. */ - ownedNames: Set; - tools: unknown[]; -} | null => { - if (!Array.isArray(tools) || !tools.length) { - return null; - } - - let matched = false; - let executes = false; - // Names the proxy is executing itself. A client may declare its own tool - // under the same name, and the loop must not answer those calls: matching - // the name is not enough to own it. - const ownedNames = new Set(); - - const rewritten = tools.flatMap((tool): unknown[] => { - if (isWebSearchTool(tool)) { - if (searchEnabled && searchProvider) { - matched = true; - executes = true; - ownedNames.add(normalizeToolName(WEB_SEARCH_TOOL_NAME)); - - return [{ type: 'function', function: buildWebSearchToolDefinition() }]; - } - - if (!isServerDeclaredSearchTool(tool)) { - return [tool]; - } - - matched = true; - return searchPassthrough ? [stripServerToolMarker(tool)] : []; - } - - if (isWebFetchTool(tool)) { - // A client-owned function of the same name wins over the backend, exactly - // as it does for search. The backend setting chooses who runs the *proxy's* - // tool; it is not a licence to take over a tool the client declared and - // resolves itself. Without this, a client that ships its own `web_fetch` - // loses it the moment a deployment picks a backend. - if (!isServerDeclaredFetchTool(tool)) { - return [tool]; - } - - if (fetchEnabled && fetchProvider) { - matched = true; - executes = true; - ownedNames.add(normalizeToolName(WEB_FETCH_TOOL_NAME)); - - return [{ type: 'function', function: buildWebFetchToolDefinition() }]; - } - - matched = true; - return [stripServerToolMarker(tool)]; - } - - // The marker is internal to this proxy, so it never reaches upstream. - return [stripServerToolMarker(tool)]; - }); - - return matched ? { executes, ownedNames, tools: rewritten } : null; -}; - -/** - * Reads one string field out of a tool-call argument object. - * - * Backends expect a single string, but models emit `query`, `q`, - * `search_query`, or an Anthropic-style `{query: {q: ...}}` nested object, so - * any string-ish value is accepted rather than failing the call. - */ -const extractStringArgument = ({ - keys, - rawArguments, - required, -}: { - keys: string[]; - rawArguments: string | undefined; - required: boolean; -}): string => { - if (!rawArguments) { - return ''; - } - - try { - const parsed = JSON.parse(rawArguments) as unknown; - - // Some clients send a bare JSON string rather than an object. - if (typeof parsed === 'string') { - return parsed.trim(); - } - - const record = asRecord(parsed); - - if (!record) { - return ''; - } - - for (const key of keys) { - const value = record[key]; - - if (typeof value === 'string' && value.trim()) { - return value.trim(); - } - - // Anthropic-style arguments nest the value one level deeper. - const nested = asRecord(value); - - if (nested) { - for (const nestedKey of keys) { - const nestedValue = nested[nestedKey]; - - if (typeof nestedValue === 'string' && nestedValue.trim()) { - return nestedValue.trim(); - } - } - } - } - - if (required) { - // Fall back to whichever field holds the first non-empty string, so an - // unexpected argument shape still yields a usable value. Only safe when - // every field means the same thing, which is true for a single-string - // search query but not for a fetch's url plus prompt. - const firstString = Object.values(record).find( - (value): value is string => - typeof value === 'string' && value.trim().length > 0, - ); - - return firstString?.trim() ?? ''; - } - - return ''; - } catch { - // Malformed JSON: treat the raw text as the value so the call still runs. - return rawArguments.trim(); - } -}; - -const extractSearchQuery = (rawArguments: string | undefined): string => - extractStringArgument({ - keys: ['query', 'q', 'search_query', 'text'], - rawArguments, - required: true, - }); - -/** - * Builds the `web_fetch` arguments. - * - * A missing URL is reported to the model rather than thrown: the model sent the - * call, so telling it the argument was missing lets it retry correctly, whereas - * an exception would surface as an opaque tool failure. - */ -const extractFetchQuery = (rawArguments: string | undefined): WebFetchQuery => { - const url = extractStringArgument({ - keys: ['url', 'uri', 'link'], - rawArguments, - required: false, - }); - const prompt = extractStringArgument({ - keys: ['prompt', 'question', 'goal'], - rawArguments, - required: false, - }); - - return { ...(prompt ? { prompt } : {}), url }; -}; - -const sumUsage = (accumulated: unknown, incoming: unknown): unknown => { - const left = asRecord(accumulated); - const right = asRecord(incoming); - - if (!left) { - return incoming ?? null; - } - - if (!right) { - return accumulated; - } - - const merged: JsonRecord = { ...left }; - - for (const [key, value] of Object.entries(right)) { - const previous = left[key]; - - if (typeof value === 'number' && typeof previous === 'number') { - merged[key] = previous + value; - } else if (value !== undefined) { - merged[key] = value; - } - } - - return merged; -}; - -/** - * Folds completed search results into the assistant text and re-emits the - * outstanding tool calls unchanged, so a turn that mixed search with - * client-side calls stays a valid transcript. The client sees its own calls - * come back as if upstream had returned them directly. - * - * The findings are folded only when the route has no other way to carry them. - * A route that renders them structurally passes - * `findingsAsStructuredBlocks`, and the text is left alone: the results are - * already on the wire as a result block, and a second copy in the prose is - * what the user reads as the model reciting its own search output. - */ -const buildMixedTurnPayload = ({ - findingsAsStructuredBlocks = false, - message, - payload, - remainingCalls, - searchResults, - usage, -}: { - findingsAsStructuredBlocks?: boolean; - message: ChatCompletionMessage | undefined; - payload: ChatCompletionPayload; - remainingCalls: ChatCompletionToolCall[]; - searchResults: string[]; - usage?: unknown; -}): ChatCompletionPayload => { - const existingText = - typeof message?.content === 'string' && message.content.trim() - ? message.content.trim() - : ''; - const findings = findingsAsStructuredBlocks - ? '' - : searchResults.filter(Boolean).join('\n\n'); - const content = [existingText, findings].filter(Boolean).join('\n\n'); - - return { - ...payload, - ...(usage ? { usage } : {}), - choices: (payload.choices ?? []).map((choice, index) => - index === 0 - ? { - ...choice, - finish_reason: 'tool_calls', - message: { - ...(choice.message ?? {}), - content: content || null, - role: 'assistant', - tool_calls: remainingCalls, - }, - } - : choice, - ), - }; -}; - -export const readReasoning = ( - message: ChatCompletionMessage | undefined, -): string => { - if (!message) { - return ''; - } - - if (typeof message.reasoning_content === 'string') { - return message.reasoning_content; - } - - return typeof message.reasoning === 'string' ? message.reasoning : ''; -}; - -/** - * Pairs each hop's reasoning and text with the calls that hop made. - * - * The three arrays are index-aligned — entry N is hop N — so zipping them back - * together is what restores the grouping a joined string cannot express. `texts` - * and `reasonings` are one entry longer than `executions`, because the closing - * hop answers instead of calling another tool. - */ -const buildIntermediateTurns = ({ - executions, - reasonings, - texts, -}: { - executions: ServerToolExecution[][]; - reasonings: string[]; - texts: string[]; -}): ServerToolTurn[] => - Array.from( - { length: Math.max(reasonings.length, texts.length) }, - (_, index) => ({ - // Only `executions` can run short: the closing hop answers without - // calling anything, so it has an entry in the prose arrays but none here. - // The three arrays stay aligned because every hop appends to all of them. - executions: executions[index] ?? [], - reasoning: reasonings[index], - text: texts[index], - }), - ); - -/** - * Folds the text a multi-hop turn produced before its later server-tool calls - * into the payload the client receives. - * - * Only the last iteration's message is in `payload`, but a turn that searched - * more than once spoke before each search, and that text is part of the turn: - * dropping it hides the model's reasoning from the user and leaves the - * client's transcript out of step with what the model actually said. - * - * The same hops are also re-grouped into the `turns` half of the result, - * because the folded strings cannot express where one hop ends and the next - * begins. That half travels beside the response rather than inside it: the - * payload is what an OpenAI-protocol client receives, and `turns` is not part - * of that protocol, so it is handed over out of band like `executions`. - * - * Shared with the image-generation loop, which has the same shape: a local - * tool call is replayed with its result appended, so only the final hop's - * message would otherwise survive. - */ -export const withIntermediateTurns = ({ - executions, - payload, - reasonings, - texts, -}: { - executions: ServerToolExecution[][]; - payload: ChatCompletionPayload; - reasonings: string[]; - texts: string[]; -}): { payload: ChatCompletionPayload; turns: ServerToolTurn[] } => { - const extraText = texts.filter(Boolean).join('\n\n'); - const extraReasoning = reasonings.filter(Boolean).join('\n\n'); - const [first, ...rest] = payload.choices ?? []; - - if (!first) { - return { payload, turns: [] }; - } - - const message = first.message ?? {}; - const existingText = - typeof message.content === 'string' ? message.content : ''; - - // Nothing from the earlier hops and nothing to fold in — but the hops may - // still have run tools, which is exactly the case a caller consuming `turns` - // needs: a hop that called a tool without speaking first is still a hop. - if (!extraText && !extraReasoning && !executions.length) { - return { payload, turns: [] }; - } - - const content = [extraText, existingText].filter(Boolean).join('\n\n'); - const reasoning = [extraReasoning, readReasoning(message)] - .filter(Boolean) - .join('\n\n'); - - return { - payload: { - ...payload, - choices: [ - { - ...first, - message: { - ...message, - content, - ...(reasoning ? { reasoning_content: reasoning } : {}), - }, - }, - ...rest, - ], - }, - // Per-hop grouping for renderers that can express it. The joined strings - // above stay as the OpenAI-shaped view; a client that builds Anthropic - // content blocks needs to know where one hop's reasoning ends and the next - // begins, which a joined string has already lost. The closing hop is the - // model's final answer, so it carries no further calls. - turns: buildIntermediateTurns({ - executions, - reasonings: [...reasonings, readReasoning(message)], - texts: [...texts, existingText], - }), - }; -}; - -/** - * Result of one server-tool pass. - * - * `response` is null when no tool could be executed: the request still has to - * be sent, but with the server-tool declarations already stripped, so the - * caller falls through to its ordinary upstream path. - */ -export interface ServerToolLoopResult { - body: ChatRequestBody; - executions: ServerToolExecution[]; - response: Response | null; - /** - * One entry per server-tool hop, in the order the model produced them. - * - * Carries the grouping `message.content` / `reasoning_content` cannot: a - * multi-hop turn joins every hop into one string per kind, which loses where - * one hop's reasoning ends and the next begins. Travels beside the response - * rather than inside it, because it is not part of the OpenAI protocol — a - * block renderer reads it off the response through `getServerToolTurns`. - * Empty when no hop ran. - */ - turns: ServerToolTurn[]; -} - -export type ServerToolInvocation = - | { - id: string; - input: { query: string }; - type: 'web_search'; - } - | { - id: string; - input: WebFetchQuery; - type: 'web_fetch'; - }; - -export type ServerToolExecution = - | (Extract & { - result: WebSearchResponse; - }) - | (Extract & { - result: WebFetchResponse; - }); - -/** - * One server-tool hop as the model produced it. - * - * `reasoning` and `text` are what the model wrote before the calls in - * `executions`; both are empty when it called tools without speaking first. The - * last hop of a turn usually has no executions, because the model answered - * instead of reaching for another tool. - */ -export interface ServerToolTurn { - executions: ServerToolExecution[]; - reasoning: string; - text: string; -} - -export interface ServerToolCallbacks { - emitStreamEvents?: boolean; - /** - * Set by routes that render a server tool's findings structurally — - * Anthropic's `web_search_tool_result` block — instead of as prose. Those - * routes must not also fold the same findings into the assistant text, or - * the user sees the results twice: once as a result block and once as if - * the model had written them. - */ - findingsAsStructuredBlocks?: boolean; - onCall?: (invocation: ServerToolInvocation) => void; - onResult?: (execution: ServerToolExecution) => void; -} - -const SERVER_TOOL_STREAM_EVENT_KEY = 'x-codebuddy2api-server-tool'; - -export type ServerToolStreamEvent = - | { invocation: ServerToolInvocation; phase: 'call' } - | { execution: ServerToolExecution; phase: 'result' }; - -export const getServerToolStreamEvent = ( - value: unknown, -): ServerToolStreamEvent | null => { - const record = asRecord(value); - const event = asRecord(record?.[SERVER_TOOL_STREAM_EVENT_KEY]); - - if (event?.phase === 'call' && event.invocation) { - return { - invocation: event.invocation as ServerToolInvocation, - phase: 'call', - }; - } - - if (event?.phase === 'result' && event.execution) { - return { - execution: event.execution as ServerToolExecution, - phase: 'result', - }; - } - - return null; -}; - -const serverToolExecutions = new WeakMap(); - -export const attachServerToolExecutions = ( - response: Response, - executions: ServerToolExecution[], -): Response => { - if (executions.length) { - serverToolExecutions.set(response, executions); - } - - return response; -}; - -export const getServerToolExecutions = ( - response: Response, -): ServerToolExecution[] => serverToolExecutions.get(response) ?? []; - -/** - * Per-hop grouping, kept off the wire for the same reason `executions` is: it - * is not part of the OpenAI protocol, so a `/v1/chat/completions` client must - * not see it — a strict validator can reject the extra field, and the tool - * data would otherwise be sent twice. - */ -const serverToolTurns = new WeakMap(); - -export const attachServerToolTurns = ( - response: Response, - turns: ServerToolTurn[], -): Response => { - if (turns.length) { - serverToolTurns.set(response, turns); - } - - return response; -}; - -export const getServerToolTurns = ( - response: Response, -): ServerToolTurn[] | undefined => serverToolTurns.get(response); - -export type ServerToolUpstreamMode = - 'buffer' | 'detect-both' | 'detect-fetch' | 'detect-search' | 'stream'; - -const mergeStreamingToolName = (previous: string, incoming: string): string => { - if (!previous || incoming.startsWith(previous)) return incoming; - if (!incoming || previous.endsWith(incoming)) return previous; - return previous + incoming; -}; - -const aggregateStreamingToolCalls = ( - deltas: ChatCompletionToolCall[], -): ChatCompletionToolCall[] => { - const calls = new Map< - string, - ChatCompletionToolCall & { - function: { arguments: string; name: string }; - } - >(); - const latestKeyByIndex = new Map(); - - deltas.forEach((delta, position) => { - const indexedKey = - typeof delta.index === 'number' - ? latestKeyByIndex.get(delta.index) - : undefined; - const key = - indexedKey ?? - (delta.id ? `id:${delta.id}` : undefined) ?? - (typeof delta.index === 'number' - ? `index:${delta.index}` - : `position:${position}`); - const current = calls.get(key) ?? { - function: { arguments: '', name: '' }, - index: delta.index, - }; - - current.id = delta.id ?? current.id; - current.index = delta.index ?? current.index; - current.type = delta.type ?? current.type; - current.function.arguments += delta.function?.arguments ?? ''; - current.function.name = mergeStreamingToolName( - current.function.name, - delta.function?.name ?? '', - ); - calls.set(key, current); - - if (typeof delta.index === 'number') { - latestKeyByIndex.set(delta.index, key); - } - }); - - return [...calls.values()]; -}; - -const buildServerToolInvocation = ( - toolCall: ChatCompletionToolCall, - iteration: number, - index: number, -): ServerToolInvocation => - isWebFetchToolCall(toolCall) - ? { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: extractFetchQuery(toolCall.function?.arguments), - type: 'web_fetch', - } - : { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: { query: extractSearchQuery(toolCall.function?.arguments) }, - type: 'web_search', - }; - -const executeServerToolInvocations = async ({ - callbacks, - fetchProvider, - invocations, - searchProvider, -}: { - callbacks?: ServerToolCallbacks; - fetchProvider: WebFetchProvider | null; - invocations: ServerToolInvocation[]; - searchProvider: WebSearchProvider | null; -}): Promise< - Array<{ - content: string; - execution: ServerToolExecution; - tool_call_id: string; - }> -> => { - invocations.forEach((invocation) => callbacks?.onCall?.(invocation)); - - return await Promise.all( - invocations.map(async (invocation) => { - if (invocation.type === 'web_fetch') { - const result = await runWebFetchResult({ - provider: fetchProvider, - query: invocation.input, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - } - - const result = await runWebSearchResult({ - provider: searchProvider, - query: invocation.input.query, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - }), - ); -}; - -/** - * Result of streaming one upstream response while watching for server-tool - * calls, so a follow-up iteration can decide what happened. - * - * `localCalls` and `remainingCalls` partition the aggregated tool calls the - * way the execution loop needs them; `frames` are the frames that were held - * back because they carried tool-call deltas. - */ -interface ServerToolProbe { - content: string; - frames: string[]; - localCalls: ChatCompletionToolCall[]; - reasoning: string; - remainingCalls: ChatCompletionToolCall[]; - role: string; - toolCalls: ChatCompletionToolCall[]; - usage: unknown; -} - -/** - * Whether the proxy is the one meant to answer this call. - * - * A call without an available backend is not a fallback to the client — it - * leaves the loop as an unanswered client-owned tool — but it must not be - * counted as locally executable either. - */ -const isLocalServerToolCall = ({ - fetchProvider, - ownedNames, - toolCall, - searchProvider, -}: { - fetchProvider: WebFetchProvider | null; - /** - * Canonical names the proxy took over. Without it a client's own tool that - * happens to share a name — `web_fetch`, which is not a server tool in the - * Responses API — gets executed by the loop instead of handed back. - */ - ownedNames?: Set; - toolCall: ChatCompletionToolCall; - searchProvider: WebSearchProvider | null; -}): boolean => - (Boolean(searchProvider) && - isWebSearchToolCall(toolCall) && - isOwned(ownedNames, WEB_SEARCH_TOOL_NAME)) || - (Boolean(fetchProvider) && - isWebFetchToolCall(toolCall) && - isOwned(ownedNames, WEB_FETCH_TOOL_NAME)); - -/** - * Whether the proxy owns calls to `name`. - * - * `undefined` means the caller predates ownership tracking; those callers only - * ever run the proxy's own declarations, so they are unaffected by client tools - * of the same name. - */ -const isOwned = (ownedNames: Set | undefined, name: string): boolean => - !ownedNames || ownedNames.has(normalizeToolName(name)); - -const probeServerToolStream = async ({ - canContinue, - context, - emitRaw, - fetchProvider, - onReader, - ownedNames, - response, - searchProvider, -}: { - canContinue: () => boolean; - context: { - responseCreated: number; - responseId: string; - responseModel: string; - responseObject: string; - role: string; - usage: unknown; - }; - emitRaw: (frame: string) => void; - fetchProvider: WebFetchProvider | null; - ownedNames?: Set; - /** - * Hands the active reader to the caller's cancellation path. Without it a - * disconnect cannot interrupt a read that is already parked: the loop only - * notices the cancellation once upstream produces another chunk, which a - * stalled upstream never does. - */ - onReader?: (reader: ReadableStreamDefaultReader | null) => void; - response: Response; - searchProvider: WebSearchProvider | null; -}): Promise => { - const frames: string[] = []; - const toolCallDeltas: ChatCompletionToolCall[] = []; - const decoder = new TextDecoder(); - const reader = response.body!.getReader(); - onReader?.(reader); - let buffer = ''; - let content = ''; - let reasoning = ''; - - const inspectFrame = (frame: string): void => { - const line = frame - .split(/\r?\n/) - .find((segment) => segment.startsWith('data:')); - - if (!line) { - emitRaw(frame); - return; - } - - const raw = line.slice(5).trim(); - if (!raw) return; - if (raw === '[DONE]') { - frames.push(frame); - return; - } - - try { - const chunk = JSON.parse(raw) as { - choices?: Array<{ - delta?: ChatCompletionMessage & { - tool_calls?: ChatCompletionToolCall[]; - }; - finish_reason?: string | null; - }>; - created?: number; - id?: string; - model?: string; - object?: string; - usage?: unknown; - }; - context.responseId = chunk.id ?? context.responseId; - context.responseModel = chunk.model ?? context.responseModel; - context.responseObject = - chunk.object?.replace(/\.chunk$/, '') ?? context.responseObject; - context.responseCreated = chunk.created ?? context.responseCreated; - context.usage = chunk.usage ?? context.usage; - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - context.role = delta?.role ?? context.role; - content += delta?.content ?? ''; - reasoning += delta?.reasoning_content ?? delta?.reasoning ?? ''; - - // A tool-call frame is held rather than forwarded: if the turn turns out - // to invoke a server tool, the call has to be answered locally instead - // of being handed to the client as an unresolved call. Anything else the - // delta carried — most importantly the text the model wrote before - // deciding to search — still belongs to the visible turn, so it is - // re-emitted without the tool call. - if (delta?.tool_calls?.length) { - toolCallDeltas.push(...delta.tool_calls); - frames.push(frame); - - const visibleDelta = { ...delta }; - delete visibleDelta.tool_calls; - - if (Object.keys(visibleDelta).length) { - const visible = JSON.stringify({ - ...chunk, - choices: [{ ...choice, delta: visibleDelta, finish_reason: null }], - }); - emitRaw(`data: ${visible}`); - } - return; - } - - if (choice?.finish_reason === 'tool_calls') { - frames.push(frame); - return; - } - } catch { - emitRaw(frame); - return; - } - - emitRaw(frame); - }; - - while (true) { - const chunk = await reader.read(); - if (!canContinue()) break; - if (chunk.done) break; - buffer += decoder.decode(chunk.value, { stream: true }); - const split = buffer.split(/\r?\n\r?\n/); - buffer = split.pop() ?? ''; - split.forEach(inspectFrame); - } - - if (buffer.trim()) inspectFrame(buffer); - reader.releaseLock(); - onReader?.(null); - - const toolCalls = aggregateStreamingToolCalls(toolCallDeltas); - const isLocalCall = (toolCall: ChatCompletionToolCall): boolean => - isLocalServerToolCall({ - fetchProvider, - ownedNames, - searchProvider, - toolCall, - }); - - return { - content, - frames, - localCalls: toolCalls.filter(isLocalCall), - reasoning, - remainingCalls: toolCalls.filter((toolCall) => !isLocalCall(toolCall)), - role: context.role, - toolCalls, - usage: context.usage, - }; -}; - const createInlineServerToolStream = async ({ body, callbacks, @@ -1335,7 +237,7 @@ const createInlineServerToolStream = async ({ object: `${responseObject}.chunk`, usage, }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -1387,7 +289,7 @@ const createInlineServerToolStream = async ({ if (!response.ok || buffered.error) { emitJson(controller, buffered as JsonRecord); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -1570,7 +472,7 @@ const createInlineServerToolStream = async ({ if (!response.ok || finalPayload.error) { emitJson(controller, finalPayload as JsonRecord); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -1821,55 +723,15 @@ export const executeWebSearchLoop = async ({ typeof message?.content === 'string' ? message.content.trim() : ''; const iterationReasoning = readReasoning(message).trim(); - const invocations = localCalls.map( - (toolCall, index): ServerToolInvocation => - isWebFetchToolCall(toolCall) - ? { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: extractFetchQuery(toolCall.function?.arguments), - type: 'web_fetch', - } - : { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: { - query: extractSearchQuery(toolCall.function?.arguments), - }, - type: 'web_search', - }, - ); - invocations.forEach((invocation) => callbacks?.onCall?.(invocation)); - - const results = await Promise.all( - invocations.map(async (invocation) => { - if (invocation.type === 'web_fetch') { - const result = await runWebFetchResult({ - provider: fetchProvider, - query: invocation.input, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - } - - const result = await runWebSearchResult({ - provider: searchProvider, - query: invocation.input.query, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - }), + const invocations = localCalls.map((toolCall, index) => + buildServerToolInvocation(toolCall, iteration, index), ); + const results = await executeServerToolInvocations({ + callbacks, + fetchProvider, + invocations, + searchProvider, + }); executions.push(...results.map((result) => result.execution)); // A turn mixing server tools with client-side calls cannot be continued @@ -2036,131 +898,27 @@ export const executeWebSearchLoop = async ({ }; }; -/** - * Replays a buffered completion as chat-completion SSE. Used only after a - * streaming request actually invokes a server-executed tool; ordinary answers - * keep the upstream response untouched. - */ -export const synthesizeChatCompletionStream = ( - payload: ChatCompletionPayload, - fallbackModel: string, -): Response => { - const encoder = new TextEncoder(); - const choice = payload.choices?.[0]; - const message = choice?.message; - const created = - typeof payload.created === 'number' - ? payload.created - : Math.floor(Date.now() / 1000); - const model = - typeof payload.model === 'string' && payload.model - ? payload.model - : fallbackModel; - const id = - typeof payload.id === 'string' && payload.id - ? payload.id - : `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`; - - const enqueue = ( - controller: ReadableStreamDefaultController, - chunk: JsonRecord, - ): void => { - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ - ...chunk, - created, - id, - model, - object: 'chat.completion.chunk', - })}\n\n`, - ), - ); - }; - - const stream = new ReadableStream({ - start: (controller) => { - enqueue(controller, { - choices: [{ delta: { role: 'assistant' }, index: 0 }], - }); - - const reasoning = message?.reasoning_content ?? message?.reasoning; - - if (reasoning) { - enqueue(controller, { - choices: [{ delta: { reasoning_content: reasoning }, index: 0 }], - }); - } - - const content = - typeof message?.content === 'string' ? message.content : ''; - - for ( - let offset = 0; - offset < content.length; - offset += STREAM_TEXT_CHUNK_LENGTH - ) { - enqueue(controller, { - choices: [ - { - delta: { - content: content.slice( - offset, - offset + STREAM_TEXT_CHUNK_LENGTH, - ), - }, - index: 0, - }, - ], - }); - } - - const passthroughCalls = (message?.tool_calls ?? []).map( - (toolCall, index) => ({ - ...toolCall, - id: toolCall.id ?? `call_${index}`, - index, - type: toolCall.type ?? 'function', - }), - ); - - if (passthroughCalls.length) { - enqueue(controller, { - choices: [{ delta: { tool_calls: passthroughCalls }, index: 0 }], - }); - } - - enqueue(controller, { - choices: [ - { - delta: {}, - finish_reason: - choice?.finish_reason ?? - (passthroughCalls.length ? 'tool_calls' : 'stop'), - index: 0, - }, - ], - }); - - if (payload.usage !== undefined && payload.usage !== null) { - enqueue(controller, { - choices: [], - usage: payload.usage, - }); - } - - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - controller.close(); - }, - }); - - return new Response(stream, { - status: 200, - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); -}; +export type { + ChatCompletionMessage, + ChatCompletionPayload, + ChatCompletionToolCall, + ServerToolCallbacks, + ServerToolExecution, + ServerToolInvocation, + ServerToolLoopResult, + ServerToolStreamEvent, + ServerToolTurn, + ServerToolUpstreamMode, +} from './server-tool/types'; + +export { + attachServerToolExecutions, + attachServerToolTurns, + getServerToolExecutions, + getServerToolStreamEvent, + getServerToolTurns, +} from './server-tool/execution'; + +export { synthesizeChatCompletionStream } from './server-tool/sse'; + +export { withIntermediateTurns } from './server-tool/turns'; diff --git a/lib/server/shared/content.ts b/lib/server/shared/content.ts new file mode 100644 index 0000000..f907f7c --- /dev/null +++ b/lib/server/shared/content.ts @@ -0,0 +1,73 @@ +/** + * Small coercion helpers shared across the proxy and domain layers. + * + * Both functions here were declared independently in several modules before + * being lifted out. Guarding with `!Array.isArray` matters: an array is an + * object, so a bare `typeof value === 'object'` test lets arrays through as + * records, which is almost never what the caller meant. + */ + +/** Narrows an unknown value to a plain object, rejecting arrays and null. */ +export const asRecord = (value: unknown): Record | null => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null; + +/** + * Flattens message content into a single string. + * + * Content arrives as a bare string, an array of parts, or something else + * entirely depending on which protocol produced it. Parts carrying a `text` + * field contribute that text; anything else falls back to its JSON form so an + * unexpected part still shows up rather than vanishing. + */ +export const stringifyContent = (value: unknown): string => { + if (typeof value === 'string') { + return value; + } + + if (value === undefined || value === null) { + return ''; + } + + if (Array.isArray(value)) { + return value + .map((item) => { + if (typeof item === 'string') { + return item; + } + + if (item && typeof item === 'object' && 'text' in item) { + return String((item as { text?: unknown }).text ?? ''); + } + + return JSON.stringify(item); + }) + .join(''); + } + + return JSON.stringify(value); +}; + +/** + * Reads the reasoning a message carries, accepting either spelling. + * + * Upstreams emit `reasoning_content` (the streaming/response field) or + * `reasoning` (the field we replay prior-turn reasoning on), so both are + * recognised with `reasoning_content` taking precedence. + */ +export const readReasoning = ( + message: { reasoning?: unknown; reasoning_content?: unknown } | undefined, +): string => { + if (!message) { + return ''; + } + + const { reasoning, reasoning_content: reasoningContent } = message; + + return typeof reasoningContent === 'string' + ? reasoningContent + : typeof reasoning === 'string' + ? reasoning + : ''; +}; diff --git a/lib/server/shared/sse.ts b/lib/server/shared/sse.ts new file mode 100644 index 0000000..3217303 --- /dev/null +++ b/lib/server/shared/sse.ts @@ -0,0 +1,57 @@ +/** + * Server-sent-event framing shared by every streaming response shape the proxy + * serves. + * + * The header block and the terminating `[DONE]` frame were previously spelled + * out inline at a dozen sites, which is how one of them ended up carrying a CORS + * header the rest did not. Keeping them here also keeps the three protocols in + * step when the framing has to change. + */ + +const SSE_HEADERS: Record = { + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream; charset=utf-8', +}; + +/** + * Headers for an SSE response. `status` defaults to 200 because a stream is + * produced once the upstream has already been accepted; callers replying with + * an upstream's status pass it explicitly. + */ +export const createSseHeaders = ( + overrides?: Record, +): Record => ({ ...SSE_HEADERS, ...overrides }); + +export const createSseResponse = ( + body: BodyInit | null, + init?: { headers?: Record; status?: number }, +): Response => + new Response(body, { + headers: createSseHeaders(init?.headers), + status: init?.status ?? 200, + }); + +const encoder = new TextEncoder(); + +/** + * Text forms, for bodies assembled by joining strings rather than by enqueuing + * chunks onto a stream. The `\n\n` terminator comes from the join, so neither + * form carries one of its own. + */ +export const eventFrameText = (type: string, data: unknown): string => + `event: ${type}\ndata: ${JSON.stringify(data)}`; + +export const DONE_FRAME_TEXT = 'data: [DONE]'; + +/** + * Serialises one named event. Used where the protocol requires an `event:` + * line — Responses and Anthropic both key their consumers off it, while the + * OpenAI chat shape omits it. + */ +export const encodeEventFrame = (type: string, data: unknown): Uint8Array => + encoder.encode(`${eventFrameText(type, data)}\n\n`); + +/** The frame that ends an SSE stream. */ +export const encodeDoneFrame = (): Uint8Array => + encoder.encode(`${DONE_FRAME_TEXT}\n\n`); diff --git a/tests/server/shared-content.test.ts b/tests/server/shared-content.test.ts new file mode 100644 index 0000000..db7b063 --- /dev/null +++ b/tests/server/shared-content.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; + +import { + asRecord, + readReasoning, + stringifyContent, +} from '@/lib/server/shared/content'; + +describe('asRecord', () => { + it('accepts a plain object', () => { + expect(asRecord({ a: 1 })).toEqual({ a: 1 }); + }); + + it('rejects everything that is not a plain object', () => { + expect(asRecord(null)).toBeNull(); + expect(asRecord(undefined)).toBeNull(); + expect(asRecord('text')).toBeNull(); + expect(asRecord(0)).toBeNull(); + expect(asRecord([])).toBeNull(); + expect(asRecord([{ a: 1 }])).toBeNull(); + }); +}); + +describe('stringifyContent', () => { + it('passes a string through untouched', () => { + expect(stringifyContent('hello')).toBe('hello'); + expect(stringifyContent('')).toBe(''); + }); + + it('reports nothing for an absent value', () => { + expect(stringifyContent(undefined)).toBe(''); + expect(stringifyContent(null)).toBe(''); + }); + + it('concatenates the text of each part of an array', () => { + expect(stringifyContent(['a', { text: 'b' }, 'c'])).toBe('abc'); + }); + + it('falls back to JSON for a part without text', () => { + expect(stringifyContent([{ url: 'https://example.com/x.png' }])).toBe( + '{"url":"https://example.com/x.png"}', + ); + }); + + it('renders an empty text part as nothing', () => { + expect(stringifyContent([{ text: undefined }])).toBe(''); + }); + + it('JSON-encodes a value of any other shape', () => { + expect(stringifyContent({ a: 1 })).toBe('{"a":1}'); + expect(stringifyContent(42)).toBe('42'); + }); +}); + +describe('readReasoning', () => { + it('prefers reasoning_content over reasoning', () => { + expect(readReasoning({ reasoning: 'soon', reasoning_content: 'now' })).toBe( + 'now', + ); + }); + + it('falls back to reasoning', () => { + expect(readReasoning({ reasoning: 'text' })).toBe('text'); + }); + + it('reports nothing when the message carries neither', () => { + expect(readReasoning({})).toBe(''); + expect(readReasoning(undefined)).toBe(''); + }); + + it('ignores reasoning of an unexpected type', () => { + expect(readReasoning({ reasoning: 42 })).toBe(''); + }); +}); diff --git a/tests/server/shared-sse.test.ts b/tests/server/shared-sse.test.ts new file mode 100644 index 0000000..c177c6c --- /dev/null +++ b/tests/server/shared-sse.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; + +import { + createSseHeaders, + createSseResponse, + DONE_FRAME_TEXT, + encodeDoneFrame, + encodeEventFrame, + eventFrameText, +} from '@/lib/server/shared/sse'; + +const decoder = new TextDecoder(); + +const decode = (chunk: Uint8Array): string => decoder.decode(chunk); + +describe('createSseHeaders', () => { + it('carries the three headers every stream needs', () => { + expect(createSseHeaders()).toEqual({ + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream; charset=utf-8', + }); + }); + + it('lets a caller add to the set without losing the rest', () => { + const headers = createSseHeaders({ 'Access-Control-Allow-Origin': '*' }); + + expect(headers['Access-Control-Allow-Origin']).toBe('*'); + expect(headers['Cache-Control']).toBe('no-cache'); + }); +}); + +describe('createSseResponse', () => { + it('defaults to 200', async () => { + const response = createSseResponse('data: x\n\n'); + + expect(response.status).toBe(200); + expect(await response.text()).toBe('data: x\n\n'); + }); + + it('passes an explicit status through, including a null-body one', () => { + const response = createSseResponse(null, { status: 204 }); + + expect(response.status).toBe(204); + expect(response.body).toBeNull(); + }); +}); + +describe('frame serialisation', () => { + it('writes a named event with its type line', () => { + const frame = decode(encodeEventFrame('response.created', { id: 'r1' })); + + expect(frame).toBe('event: response.created\ndata: {"id":"r1"}\n\n'); + }); + + it('writes the terminating frame', () => { + expect(decode(encodeDoneFrame())).toBe('data: [DONE]\n\n'); + }); + + it('omits the terminator from the text forms, which are joined with it', () => { + expect(eventFrameText('response.created', { id: 'r1' })).toBe( + 'event: response.created\ndata: {"id":"r1"}', + ); + expect(DONE_FRAME_TEXT).toBe('data: [DONE]'); + expect([eventFrameText('a', {}), DONE_FRAME_TEXT, ''].join('\n\n')).toBe( + 'event: a\ndata: {}\n\ndata: [DONE]\n\n', + ); + }); +});