From 2280bae7df2b39a01206cdbd50bed5bd7a4d80e8 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 21:14:44 +0800 Subject: [PATCH 1/7] refactor(proxy): lift duplicated SSE and content helpers into shared modules The proxy layer carried several independent copies of the same primitives: three stringifyContent implementations, three asRecord guards, twelve inline SSE response-header blocks, and fifteen spelled-out [DONE] frames. They had already drifted - one SSE block carried a CORS header the rest did not. Move each to lib/server/shared and point every caller at it. --- lib/server/domain/account-status.ts | 6 +-- lib/server/domain/debug.ts | 7 +-- lib/server/proxy/anthropic.ts | 56 ++------------------- lib/server/proxy/codebuddy.ts | 76 ++++++++++------------------- lib/server/proxy/responses.ts | 72 ++++++--------------------- lib/server/proxy/web-search-loop.ts | 39 +++------------ lib/server/shared/content.ts | 73 +++++++++++++++++++++++++++ lib/server/shared/sse.ts | 57 ++++++++++++++++++++++ 8 files changed, 186 insertions(+), 200 deletions(-) create mode 100644 lib/server/shared/content.ts create mode 100644 lib/server/shared/sse.ts diff --git a/lib/server/domain/account-status.ts b/lib/server/domain/account-status.ts index 0fce907..5fbb66b 100644 --- a/lib/server/domain/account-status.ts +++ b/lib/server/domain/account-status.ts @@ -5,6 +5,7 @@ import { type CredentialRecord, } from './credentials'; import { getModelsForCredential } from '../proxy/codebuddy'; +import { asRecord } from '../shared/content'; export interface AccountStatusSnapshot { checkin: { claimed: boolean | null; message: string | null }; @@ -26,11 +27,6 @@ const getBearerToken = (credential: CredentialRecord): string => credential.data.bearer_token ?? credential.data.access_token ?? '', ).trim(); -const asRecord = (value: unknown): Record | null => - value && typeof value === 'object' - ? (value as Record) - : null; - const findValue = (value: unknown, keys: string[]): unknown => { const record = asRecord(value); if (record) { diff --git a/lib/server/domain/debug.ts b/lib/server/domain/debug.ts index c05dbbf..192a3c1 100644 --- a/lib/server/domain/debug.ts +++ b/lib/server/domain/debug.ts @@ -9,6 +9,7 @@ import { trimStorageDebugLogs, writeStorageJson, } from '../storage'; +import { asRecord } from '../shared/content'; export interface DebugLogEntry { credentialFilename: string | null; @@ -716,12 +717,6 @@ const captureIndependentResponseSnapshot = async ( }; }; -const asRecord = (value: unknown): Record | null => { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as Record) - : null; -}; - const toTokenCount = (value: unknown): number => { const numeric = typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index 8bbc7f2..97dc6dc 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -20,7 +20,9 @@ import { createStreamCloser, toUpstreamTimeoutMessage, } from '../shared/upstream-timeout'; +import { stringifyContent } from '../shared/content'; import { extractErrorMessage } from '../shared/http'; +import { createSseResponse } from '../shared/sse'; import { markServerTool, normalizeToolName, @@ -194,34 +196,6 @@ const createAnthropicId = (prefix: string): string => { return `${prefix}_${crypto.randomUUID().replaceAll('-', '')}`; }; -const stringifyContent = (value: unknown): string => { - if (typeof value === 'string') { - return value; - } - - if (Array.isArray(value)) { - return value - .map((item) => { - if (typeof item === 'string') { - return item; - } - - if (item && typeof item === 'object' && 'text' in item) { - return String((item as { text?: unknown }).text ?? ''); - } - - return JSON.stringify(item); - }) - .join(''); - } - - if (value === undefined || value === null) { - return ''; - } - - return JSON.stringify(value); -}; - const mapTextPartsToChatContent = ( parts: Array, ): ChatTextContent => { @@ -1059,14 +1033,7 @@ const mapOpenAIStreamToAnthropicSSE = ( }, ): Response => { if (!upstreamResponse.body) { - return new Response(null, { - status: upstreamResponse.status, - headers: { - 'Content-Type': 'text/event-stream; charset=utf-8', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }, - }); + return createSseResponse(null, { status: upstreamResponse.status }); } const encoder = new TextEncoder(); @@ -1531,14 +1498,7 @@ const mapOpenAIStreamToAnthropicSSE = ( }, }); - return new Response(stream, { - status: 200, - headers: { - 'Content-Type': 'text/event-stream; charset=utf-8', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }, - }); + return createSseResponse(stream, { status: 200 }); }; const createAnthropicServerToolEventStream = ( @@ -1659,13 +1619,7 @@ const createAnthropicServerToolEventStream = ( }, }); - return new Response(stream, { - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); + return createSseResponse(stream); }; const getUpstreamErrorMessage = async (response: Response): Promise => { diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index e8f7a12..ee3916d 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -51,6 +51,16 @@ import { toUpstreamTimeoutMessage, } from '../shared/upstream-timeout'; import { recordUsageEvent, type UsageSnapshot } from '../domain/usage'; +import { stringifyContent } from '../shared/content'; +import { createSseResponse, encodeDoneFrame } from '../shared/sse'; + +/** + * Chat completions are called from browser-side clients as well as servers, so + * this route's streams carry the CORS header the other protocols do not need. + */ +const CORS_HEADERS: Record = { + 'Access-Control-Allow-Origin': '*', +}; interface OpenAIMessage { role?: string; @@ -948,30 +958,13 @@ export const extractImageUrl = (part: unknown): string | undefined => { return undefined; }; -const stringifyResponsesInputContent = (content: unknown): string => { - if (typeof content === 'string') return content; - if (content === null || content === undefined) return ''; - if (Array.isArray(content)) { - return content - .map((part) => { - if (typeof part === 'string') return part; - if (part && typeof part === 'object' && 'text' in part) { - return String((part as { text?: unknown }).text ?? ''); - } - return JSON.stringify(part); - }) - .join(''); - } - return JSON.stringify(content); -}; - const mapChatContentToResponses = ( content: unknown, ): Array> => { if (!Array.isArray(content)) { return [ { - text: stringifyResponsesInputContent(content), + text: stringifyContent(content), type: 'input_text', }, ]; @@ -1163,9 +1156,7 @@ const normalizeResponsesUpstreamBody = async ( ); }) .map((message) => { - return stringifyResponsesInputContent( - (message as { content?: unknown }).content, - ); + return stringifyContent((message as { content?: unknown }).content); }) .filter(Boolean) .join('\n\n'); @@ -1202,7 +1193,7 @@ const buildResponsesBodyFromChat = async ( ?.filter( (message) => message.role === 'system' || message.role === 'developer', ) - .map((message) => stringifyResponsesInputContent(message.content)) + .map((message) => stringifyContent(message.content)) .filter(Boolean) .join('\n\n'); const input = @@ -1229,7 +1220,7 @@ const buildResponsesBodyFromChat = async ( call_id: message.tool_call_id, output: hasImage ? mapChatContentToResponses(toolOutput) - : stringifyResponsesInputContent(toolOutput), + : stringifyContent(toolOutput), type: 'function_call_output', }; } @@ -1627,7 +1618,7 @@ const mapResponsesStreamToChat = ( ); } enqueueUsage(controller); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); await recordStreamUsage(); reader.releaseLock(); reader = null; @@ -1643,7 +1634,7 @@ const mapResponsesStreamToChat = ( 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', ), ); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); await cancelAndReleaseReader(); await recordStreamUsage(); controller.close(); @@ -1657,7 +1648,7 @@ const mapResponsesStreamToChat = ( 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', ), ); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); await cancelAndReleaseReader(); await recordStreamUsage(); controller.close(); @@ -1877,7 +1868,7 @@ const mapResponsesStreamToChat = ( emittedFinish = true; emitted = true; if (stoppedLocally) { - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); await cancelAndReleaseReader('Stop sequence matched'); await recordStreamUsage(); controller.close(); @@ -1919,7 +1910,7 @@ const mapResponsesStreamToChat = ( emittedFinish = true; emitted = true; if (stoppedLocally) { - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); await cancelAndReleaseReader('Stop sequence matched'); await recordStreamUsage(); controller.close(); @@ -1948,7 +1939,7 @@ const mapResponsesStreamToChat = ( `data: ${JSON.stringify({ error: { message } })}\n\n`, ), ); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); await cancelAndReleaseReader(); await recordStreamUsage(); controller.close(); @@ -1968,13 +1959,8 @@ const mapResponsesStreamToChat = ( }, }); - return new Response(stream, { - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, + return createSseResponse(stream, { + headers: CORS_HEADERS, status: upstreamResponse.status, }); }; @@ -2162,14 +2148,9 @@ const normalizeStreamingResponse = ({ upstreamResponse: Response; }): Response => { if (!upstreamResponse.body) { - return new Response(null, { + return createSseResponse(null, { + headers: CORS_HEADERS, status: upstreamResponse.status, - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, }); } @@ -2308,14 +2289,9 @@ const normalizeStreamingResponse = ({ }, }); - return new Response(stream, { + return createSseResponse(stream, { + headers: CORS_HEADERS, status: upstreamResponse.status, - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, }); }; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 5f7f046..9b7f5b5 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -5,6 +5,13 @@ import { isWebFetchEnabled, isWebSearchEnabled, } from '../domain/config'; +import { stringifyContent } from '../shared/content'; +import { + createSseResponse, + DONE_FRAME_TEXT, + encodeDoneFrame, + eventFrameText, +} from '../shared/sse'; import { getCredentialSupportedModels } from '../domain/credentials'; import type { DebugTrace } from '../domain/debug'; import { @@ -871,34 +878,6 @@ const mapInputContentToTranscriptContent = ( return mapped.length ? mapped : null; }; -const stringifyContent = (value: unknown): string => { - if (typeof value === 'string') { - return value; - } - - if (Array.isArray(value)) { - return value - .map((item) => { - if (typeof item === 'string') { - return item; - } - - if (item && typeof item === 'object' && 'text' in item) { - return String((item as { text?: unknown }).text ?? ''); - } - - return JSON.stringify(item); - }) - .join(''); - } - - if (value === undefined || value === null) { - return ''; - } - - return JSON.stringify(value); -}; - /** * Marks an `encrypted_content` value we minted, so we can tell it apart from a * blob issued by someone else. @@ -1775,20 +1754,12 @@ const mapChatResponseToResponsesStream = async ( }); const body = [ - ...frames.map( - (frame) => `event: ${frame.type}\ndata: ${JSON.stringify(frame)}`, - ), - 'data: [DONE]', + ...frames.map((frame) => eventFrameText(String(frame.type), frame)), + DONE_FRAME_TEXT, '', ].join('\n\n'); - return new Response(body, { - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); + return createSseResponse(body); }; const mapChatStreamToResponsesEventStream = ( @@ -2091,7 +2062,7 @@ const mapChatStreamToResponsesEventStream = ( type: 'response.error', error: { message: 'Failed to persist response session' }, }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); releaseReader(); controller.close(); return; @@ -2186,7 +2157,7 @@ const mapChatStreamToResponsesEventStream = ( .map(({ item }) => item), }, }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); releaseReader(); controller.close(); return; @@ -2451,14 +2422,7 @@ const mapChatStreamToResponsesEventStream = ( }, }); - return new Response(stream, { - status: 200, - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); + return createSseResponse(stream, { status: 200 }); }; const createResponsesEventStream = async ( @@ -2701,7 +2665,7 @@ const createResponsesEventStream = async ( type: 'response.error', error: { message: 'Upstream request failed' }, }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -2747,13 +2711,7 @@ const createResponsesEventStream = async ( }, }); - return new Response(stream, { - headers: { - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, - }); + return createSseResponse(stream); }; export const handleResponsesRequest = async ( diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index 55d4518..35e39c3 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -10,7 +10,9 @@ import { runWebFetchResult, runWebSearchResult, } from '../search'; +import { asRecord, readReasoning } from '../shared/content'; import { extractErrorMessage } from '../shared/http'; +import { createSseResponse, encodeDoneFrame } from '../shared/sse'; import type { ChatRequestBody } from './codebuddy'; import { @@ -187,12 +189,6 @@ const readBufferedChatCompletionPayload = async ( return payload; }; -const asRecord = (value: unknown): JsonRecord | null => { - return value && typeof value === 'object' && !Array.isArray(value) - ? (value as JsonRecord) - : null; -}; - /** * Recognises one server-tool declaration. * @@ -563,20 +559,6 @@ const buildMixedTurnPayload = ({ }; }; -export const readReasoning = ( - message: ChatCompletionMessage | undefined, -): string => { - if (!message) { - return ''; - } - - if (typeof message.reasoning_content === 'string') { - return message.reasoning_content; - } - - return typeof message.reasoning === 'string' ? message.reasoning : ''; -}; - /** * Pairs each hop's reasoning and text with the calls that hop made. * @@ -1335,7 +1317,7 @@ const createInlineServerToolStream = async ({ object: `${responseObject}.chunk`, usage, }); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -1387,7 +1369,7 @@ const createInlineServerToolStream = async ({ if (!response.ok || buffered.error) { emitJson(controller, buffered as JsonRecord); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -1570,7 +1552,7 @@ const createInlineServerToolStream = async ({ if (!response.ok || finalPayload.error) { emitJson(controller, finalPayload as JsonRecord); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); return; } @@ -2149,18 +2131,13 @@ export const synthesizeChatCompletionStream = ( }); } - controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.enqueue(encodeDoneFrame()); controller.close(); }, }); - return new Response(stream, { + return createSseResponse(stream, { + headers: { 'Access-Control-Allow-Origin': '*' }, status: 200, - headers: { - 'Access-Control-Allow-Origin': '*', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - 'Content-Type': 'text/event-stream; charset=utf-8', - }, }); }; diff --git a/lib/server/shared/content.ts b/lib/server/shared/content.ts new file mode 100644 index 0000000..f907f7c --- /dev/null +++ b/lib/server/shared/content.ts @@ -0,0 +1,73 @@ +/** + * Small coercion helpers shared across the proxy and domain layers. + * + * Both functions here were declared independently in several modules before + * being lifted out. Guarding with `!Array.isArray` matters: an array is an + * object, so a bare `typeof value === 'object'` test lets arrays through as + * records, which is almost never what the caller meant. + */ + +/** Narrows an unknown value to a plain object, rejecting arrays and null. */ +export const asRecord = (value: unknown): Record | null => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null; + +/** + * Flattens message content into a single string. + * + * Content arrives as a bare string, an array of parts, or something else + * entirely depending on which protocol produced it. Parts carrying a `text` + * field contribute that text; anything else falls back to its JSON form so an + * unexpected part still shows up rather than vanishing. + */ +export const stringifyContent = (value: unknown): string => { + if (typeof value === 'string') { + return value; + } + + if (value === undefined || value === null) { + return ''; + } + + if (Array.isArray(value)) { + return value + .map((item) => { + if (typeof item === 'string') { + return item; + } + + if (item && typeof item === 'object' && 'text' in item) { + return String((item as { text?: unknown }).text ?? ''); + } + + return JSON.stringify(item); + }) + .join(''); + } + + return JSON.stringify(value); +}; + +/** + * Reads the reasoning a message carries, accepting either spelling. + * + * Upstreams emit `reasoning_content` (the streaming/response field) or + * `reasoning` (the field we replay prior-turn reasoning on), so both are + * recognised with `reasoning_content` taking precedence. + */ +export const readReasoning = ( + message: { reasoning?: unknown; reasoning_content?: unknown } | undefined, +): string => { + if (!message) { + return ''; + } + + const { reasoning, reasoning_content: reasoningContent } = message; + + return typeof reasoningContent === 'string' + ? reasoningContent + : typeof reasoning === 'string' + ? reasoning + : ''; +}; diff --git a/lib/server/shared/sse.ts b/lib/server/shared/sse.ts new file mode 100644 index 0000000..3217303 --- /dev/null +++ b/lib/server/shared/sse.ts @@ -0,0 +1,57 @@ +/** + * Server-sent-event framing shared by every streaming response shape the proxy + * serves. + * + * The header block and the terminating `[DONE]` frame were previously spelled + * out inline at a dozen sites, which is how one of them ended up carrying a CORS + * header the rest did not. Keeping them here also keeps the three protocols in + * step when the framing has to change. + */ + +const SSE_HEADERS: Record = { + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream; charset=utf-8', +}; + +/** + * Headers for an SSE response. `status` defaults to 200 because a stream is + * produced once the upstream has already been accepted; callers replying with + * an upstream's status pass it explicitly. + */ +export const createSseHeaders = ( + overrides?: Record, +): Record => ({ ...SSE_HEADERS, ...overrides }); + +export const createSseResponse = ( + body: BodyInit | null, + init?: { headers?: Record; status?: number }, +): Response => + new Response(body, { + headers: createSseHeaders(init?.headers), + status: init?.status ?? 200, + }); + +const encoder = new TextEncoder(); + +/** + * Text forms, for bodies assembled by joining strings rather than by enqueuing + * chunks onto a stream. The `\n\n` terminator comes from the join, so neither + * form carries one of its own. + */ +export const eventFrameText = (type: string, data: unknown): string => + `event: ${type}\ndata: ${JSON.stringify(data)}`; + +export const DONE_FRAME_TEXT = 'data: [DONE]'; + +/** + * Serialises one named event. Used where the protocol requires an `event:` + * line — Responses and Anthropic both key their consumers off it, while the + * OpenAI chat shape omits it. + */ +export const encodeEventFrame = (type: string, data: unknown): Uint8Array => + encoder.encode(`${eventFrameText(type, data)}\n\n`); + +/** The frame that ends an SSE stream. */ +export const encodeDoneFrame = (): Uint8Array => + encoder.encode(`${DONE_FRAME_TEXT}\n\n`); From 41e8c115bba8c14e7d3e7ac7aecbca3c3354342c Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 22:00:11 +0800 Subject: [PATCH 2/7] refactor(web-search): split the server-tool loop into focused modules web-search-loop.ts was 2143 lines holding eight unrelated concerns. Extract the wire types, declaration classification, argument parsing, turn folding, provider invocation, response buffering, stream probing and SSE synthesis into their own modules, leaving the two loops as the only thing the original file owns. Also drops an inlined copy of the invocation-building and dispatch logic that executeWebSearchLoop carried alongside the shared buildServerToolInvocation/executeServerToolInvocations helpers; the two were identical, including id format, key order and callback ordering. --- lib/server/proxy/server-tool-args.ts | 108 ++ lib/server/proxy/server-tool-classify.ts | 243 ++++ lib/server/proxy/server-tool-execution.ts | 146 +++ lib/server/proxy/server-tool-payload.ts | 98 ++ lib/server/proxy/server-tool-sse.ts | 130 ++ lib/server/proxy/server-tool-stream.ts | 233 ++++ lib/server/proxy/server-tool-turns.ts | 205 +++ lib/server/proxy/server-tool-types.ts | 129 ++ lib/server/proxy/web-search-loop.ts | 1369 ++------------------- 9 files changed, 1367 insertions(+), 1294 deletions(-) create mode 100644 lib/server/proxy/server-tool-args.ts create mode 100644 lib/server/proxy/server-tool-classify.ts create mode 100644 lib/server/proxy/server-tool-execution.ts create mode 100644 lib/server/proxy/server-tool-payload.ts create mode 100644 lib/server/proxy/server-tool-sse.ts create mode 100644 lib/server/proxy/server-tool-stream.ts create mode 100644 lib/server/proxy/server-tool-turns.ts create mode 100644 lib/server/proxy/server-tool-types.ts diff --git a/lib/server/proxy/server-tool-args.ts b/lib/server/proxy/server-tool-args.ts new file mode 100644 index 0000000..5741ebd --- /dev/null +++ b/lib/server/proxy/server-tool-args.ts @@ -0,0 +1,108 @@ +import { asRecord } from '../shared/content'; +import type { WebFetchQuery } from '../search/types'; + +/** + * Reads one string field out of a tool-call argument object. + * + * Backends expect a single string, but models emit `query`, `q`, + * `search_query`, or an Anthropic-style `{query: {q: ...}}` nested object, so + * any string-ish value is accepted rather than failing the call. + */ +export const extractStringArgument = ({ + keys, + rawArguments, + required, +}: { + keys: string[]; + rawArguments: string | undefined; + required: boolean; +}): string => { + if (!rawArguments) { + return ''; + } + + try { + const parsed = JSON.parse(rawArguments) as unknown; + + // Some clients send a bare JSON string rather than an object. + if (typeof parsed === 'string') { + return parsed.trim(); + } + + const record = asRecord(parsed); + + if (!record) { + return ''; + } + + for (const key of keys) { + const value = record[key]; + + if (typeof value === 'string' && value.trim()) { + return value.trim(); + } + + // Anthropic-style arguments nest the value one level deeper. + const nested = asRecord(value); + + if (nested) { + for (const nestedKey of keys) { + const nestedValue = nested[nestedKey]; + + if (typeof nestedValue === 'string' && nestedValue.trim()) { + return nestedValue.trim(); + } + } + } + } + + if (required) { + // Fall back to whichever field holds the first non-empty string, so an + // unexpected argument shape still yields a usable value. Only safe when + // every field means the same thing, which is true for a single-string + // search query but not for a fetch's url plus prompt. + const firstString = Object.values(record).find( + (value): value is string => + typeof value === 'string' && value.trim().length > 0, + ); + + return firstString?.trim() ?? ''; + } + + return ''; + } catch { + // Malformed JSON: treat the raw text as the value so the call still runs. + return rawArguments.trim(); + } +}; + +export const extractSearchQuery = (rawArguments: string | undefined): string => + extractStringArgument({ + keys: ['query', 'q', 'search_query', 'text'], + rawArguments, + required: true, + }); + +/** + * Builds the `web_fetch` arguments. + * + * A missing URL is reported to the model rather than thrown: the model sent the + * call, so telling it the argument was missing lets it retry correctly, whereas + * an exception would surface as an opaque tool failure. + */ +export const extractFetchQuery = ( + rawArguments: string | undefined, +): WebFetchQuery => { + const url = extractStringArgument({ + keys: ['url', 'uri', 'link'], + rawArguments, + required: false, + }); + const prompt = extractStringArgument({ + keys: ['prompt', 'question', 'goal'], + rawArguments, + required: false, + }); + + return { ...(prompt ? { prompt } : {}), url }; +}; diff --git a/lib/server/proxy/server-tool-classify.ts b/lib/server/proxy/server-tool-classify.ts new file mode 100644 index 0000000..45daa2d --- /dev/null +++ b/lib/server/proxy/server-tool-classify.ts @@ -0,0 +1,243 @@ +import { asRecord } from '../shared/content'; +import { + buildWebFetchToolDefinition, + buildWebSearchToolDefinition, + isMarkedServerTool, + normalizeToolName, + stripServerToolMarker, + WEB_FETCH_TOOL_NAME, + WEB_FETCH_TOOL_TYPE_PREFIX, + WEB_SEARCH_TOOL_NAME, + WEB_SEARCH_TOOL_TYPE_PREFIX, +} from '../search/tool'; +import type { WebFetchProvider, WebSearchProvider } from '../search/types'; +import type { ChatCompletionToolCall } from './server-tool-types'; + +/** + * Recognises one server-tool declaration. + * + * Anthropic sends dated server-tool *types* (`web_search_20260209`, + * `web_fetch_20250910`), Responses sends `web_search_preview`, and a client may + * also declare a plain function tool with the bare name for its own purposes. + * All three shapes have to match, because the tool has to be swapped for a + * function upstream can actually call regardless of how it arrived. + * + * Returns two independent answers. `matches` says the declaration is one this + * proxy can serve; `serverDeclared` says it arrived as a provider-executed + * server tool rather than as the client's own function. The difference decides + * what happens when the tool cannot be executed: a server-tool declaration is + * dropped, because upstream has no idea what to do with it, whereas the + * client's own function is left exactly as sent — the client is the one that + * resolves it, and deleting it would silently remove a capability the client + * asked for. + */ +export const classifyServerTool = ( + tool: unknown, + name: string, + prefix: string, +): { matches: boolean; serverDeclared: boolean } => { + const record = asRecord(tool); + + if (!record) { + return { matches: false, serverDeclared: false }; + } + + const type = typeof record.type === 'string' ? record.type : ''; + + // A dedicated server-tool type (`web_search_20260209`, `web_fetch_20250910`, + // `web_search_preview`) is unambiguous: only a provider-executed tool is + // declared that way. The trailing date is part of the version, not the name, + // so the prefix is matched in canonical form — `WebFetch_20250910` arrives + // from upstream as readily as its snake_case spelling. + if (normalizeToolName(type).startsWith(normalizeToolName(prefix))) { + return { matches: true, serverDeclared: true }; + } + + const fn = asRecord(record.function); + const isBareName = + (typeof fn?.name === 'string' && + normalizeToolName(fn.name) === normalizeToolName(name)) || + (typeof record.name === 'string' && + normalizeToolName(record.name).startsWith(normalizeToolName(prefix))); + + // A Responses translation has already flattened the declaration into a plain + // function, so the type is gone by now; its marker is the only surviving + // evidence that the client asked for a provider-executed tool. + return { + matches: isBareName, + serverDeclared: isBareName && isMarkedServerTool(tool), + }; +}; + +export const isWebSearchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) + .matches; + +export const isWebFetchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) + .matches; + +export const isServerDeclaredSearchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) + .serverDeclared; + +export const isServerDeclaredFetchTool = (tool: unknown): boolean => + classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) + .serverDeclared; + +/** + * Whether `toolCall` is a call the proxy is meant to execute. + * + * Matched in canonical form because the name comes back from the model, which + * is under no obligation to repeat the spelling it was given: upstream echoes + * `web_fetch` as `WebFetch` often enough to matter here. A miss is not a + * fallback to the client — the call leaves the loop as an unanswered + * client-owned tool, so the fetch silently never happens. + */ +export const isWebSearchToolCall = ( + toolCall: ChatCompletionToolCall, +): boolean => { + return ( + typeof toolCall.function?.name === 'string' && + normalizeToolName(toolCall.function.name) === + normalizeToolName(WEB_SEARCH_TOOL_NAME) + ); +}; + +export const isWebFetchToolCall = ( + toolCall: ChatCompletionToolCall, +): boolean => { + return ( + typeof toolCall.function?.name === 'string' && + normalizeToolName(toolCall.function.name) === + normalizeToolName(WEB_FETCH_TOOL_NAME) + ); +}; + +/** + * Swaps locally executed server-tool declarations for functions upstream can + * call. Passthrough tools keep their upstream representation. + * + * Returns `null` when no web tool is present. `executes` distinguishes a local + * backend from passthrough: the latter still strips the internal provenance + * marker, but never starts the server loop or buffers a stream. + */ +export const replaceServerTools = ({ + fetchEnabled, + fetchProvider, + searchEnabled, + searchPassthrough, + searchProvider, + tools, +}: { + fetchEnabled: boolean; + fetchProvider: WebFetchProvider | null; + searchEnabled: boolean; + searchPassthrough: boolean; + searchProvider: WebSearchProvider | null; + tools: unknown; +}): { + executes: boolean; + /** Canonical names the proxy took over, so call classification can tell its own calls from a client's. */ + ownedNames: Set; + tools: unknown[]; +} | null => { + if (!Array.isArray(tools) || !tools.length) { + return null; + } + + let matched = false; + let executes = false; + // Names the proxy is executing itself. A client may declare its own tool + // under the same name, and the loop must not answer those calls: matching + // the name is not enough to own it. + const ownedNames = new Set(); + + const rewritten = tools.flatMap((tool): unknown[] => { + if (isWebSearchTool(tool)) { + if (searchEnabled && searchProvider) { + matched = true; + executes = true; + ownedNames.add(normalizeToolName(WEB_SEARCH_TOOL_NAME)); + + return [{ type: 'function', function: buildWebSearchToolDefinition() }]; + } + + if (!isServerDeclaredSearchTool(tool)) { + return [tool]; + } + + matched = true; + return searchPassthrough ? [stripServerToolMarker(tool)] : []; + } + + if (isWebFetchTool(tool)) { + // A client-owned function of the same name wins over the backend, exactly + // as it does for search. The backend setting chooses who runs the *proxy's* + // tool; it is not a licence to take over a tool the client declared and + // resolves itself. Without this, a client that ships its own `web_fetch` + // loses it the moment a deployment picks a backend. + if (!isServerDeclaredFetchTool(tool)) { + return [tool]; + } + + if (fetchEnabled && fetchProvider) { + matched = true; + executes = true; + ownedNames.add(normalizeToolName(WEB_FETCH_TOOL_NAME)); + + return [{ type: 'function', function: buildWebFetchToolDefinition() }]; + } + + matched = true; + return [stripServerToolMarker(tool)]; + } + + // The marker is internal to this proxy, so it never reaches upstream. + return [stripServerToolMarker(tool)]; + }); + + return matched ? { executes, ownedNames, tools: rewritten } : null; +}; + +/** + * Whether the proxy is the one meant to answer this call. + * + * A call without an available backend is not a fallback to the client — it + * leaves the loop as an unanswered client-owned tool — but it must not be + * counted as locally executable either. + */ +export const isLocalServerToolCall = ({ + fetchProvider, + ownedNames, + toolCall, + searchProvider, +}: { + fetchProvider: WebFetchProvider | null; + /** + * Canonical names the proxy took over. Without it a client's own tool that + * happens to share a name — `web_fetch`, which is not a server tool in the + * Responses API — gets executed by the loop instead of handed back. + */ + ownedNames?: Set; + toolCall: ChatCompletionToolCall; + searchProvider: WebSearchProvider | null; +}): boolean => + (Boolean(searchProvider) && + isWebSearchToolCall(toolCall) && + isOwned(ownedNames, WEB_SEARCH_TOOL_NAME)) || + (Boolean(fetchProvider) && + isWebFetchToolCall(toolCall) && + isOwned(ownedNames, WEB_FETCH_TOOL_NAME)); + +/** + * Whether the proxy owns calls to `name`. + * + * `undefined` means the caller predates ownership tracking; those callers only + * ever run the proxy's own declarations, so they are unaffected by client tools + * of the same name. + */ +export const isOwned = ( + ownedNames: Set | undefined, + name: string, +): boolean => !ownedNames || ownedNames.has(normalizeToolName(name)); diff --git a/lib/server/proxy/server-tool-execution.ts b/lib/server/proxy/server-tool-execution.ts new file mode 100644 index 0000000..af047e1 --- /dev/null +++ b/lib/server/proxy/server-tool-execution.ts @@ -0,0 +1,146 @@ +import { runWebFetchResult, runWebSearchResult } from '../search'; +import type { WebFetchProvider, WebSearchProvider } from '../search/types'; +import { asRecord } from '../shared/content'; +import { extractFetchQuery, extractSearchQuery } from './server-tool-args'; +import { isWebFetchToolCall } from './server-tool-classify'; +import { + SERVER_TOOL_STREAM_EVENT_KEY, + type ChatCompletionToolCall, + type ServerToolCallbacks, + type ServerToolExecution, + type ServerToolInvocation, + type ServerToolStreamEvent, + type ServerToolTurn, +} from './server-tool-types'; + +export const getServerToolStreamEvent = ( + value: unknown, +): ServerToolStreamEvent | null => { + const record = asRecord(value); + const event = asRecord(record?.[SERVER_TOOL_STREAM_EVENT_KEY]); + + if (event?.phase === 'call' && event.invocation) { + return { + invocation: event.invocation as ServerToolInvocation, + phase: 'call', + }; + } + + if (event?.phase === 'result' && event.execution) { + return { + execution: event.execution as ServerToolExecution, + phase: 'result', + }; + } + + return null; +}; + +const serverToolExecutions = new WeakMap(); + +export const attachServerToolExecutions = ( + response: Response, + executions: ServerToolExecution[], +): Response => { + if (executions.length) { + serverToolExecutions.set(response, executions); + } + + return response; +}; + +export const getServerToolExecutions = ( + response: Response, +): ServerToolExecution[] => serverToolExecutions.get(response) ?? []; + +/** + * Per-hop grouping, kept off the wire for the same reason `executions` is: it + * is not part of the OpenAI protocol, so a `/v1/chat/completions` client must + * not see it — a strict validator can reject the extra field, and the tool + * data would otherwise be sent twice. + */ +const serverToolTurns = new WeakMap(); + +export const attachServerToolTurns = ( + response: Response, + turns: ServerToolTurn[], +): Response => { + if (turns.length) { + serverToolTurns.set(response, turns); + } + + return response; +}; + +export const getServerToolTurns = ( + response: Response, +): ServerToolTurn[] | undefined => serverToolTurns.get(response); + +export const buildServerToolInvocation = ( + toolCall: ChatCompletionToolCall, + iteration: number, + index: number, +): ServerToolInvocation => + isWebFetchToolCall(toolCall) + ? { + id: toolCall.id ?? `server_tool_${iteration}_${index}`, + input: extractFetchQuery(toolCall.function?.arguments), + type: 'web_fetch', + } + : { + id: toolCall.id ?? `server_tool_${iteration}_${index}`, + input: { query: extractSearchQuery(toolCall.function?.arguments) }, + type: 'web_search', + }; + +export const executeServerToolInvocations = async ({ + callbacks, + fetchProvider, + invocations, + searchProvider, +}: { + callbacks?: ServerToolCallbacks; + fetchProvider: WebFetchProvider | null; + invocations: ServerToolInvocation[]; + searchProvider: WebSearchProvider | null; +}): Promise< + Array<{ + content: string; + execution: ServerToolExecution; + tool_call_id: string; + }> +> => { + invocations.forEach((invocation) => callbacks?.onCall?.(invocation)); + + return await Promise.all( + invocations.map(async (invocation) => { + if (invocation.type === 'web_fetch') { + const result = await runWebFetchResult({ + provider: fetchProvider, + query: invocation.input, + }); + const execution: ServerToolExecution = { ...invocation, result }; + callbacks?.onResult?.(execution); + + return { + content: result.content, + execution, + tool_call_id: invocation.id, + }; + } + + const result = await runWebSearchResult({ + provider: searchProvider, + query: invocation.input.query, + }); + const execution: ServerToolExecution = { ...invocation, result }; + callbacks?.onResult?.(execution); + + return { + content: result.content, + execution, + tool_call_id: invocation.id, + }; + }), + ); +}; diff --git a/lib/server/proxy/server-tool-payload.ts b/lib/server/proxy/server-tool-payload.ts new file mode 100644 index 0000000..0538e01 --- /dev/null +++ b/lib/server/proxy/server-tool-payload.ts @@ -0,0 +1,98 @@ +import { extractErrorMessage } from '../shared/http'; +import type { ChatCompletionPayload } from './server-tool-types'; + +/** + * Rebuilds a failed upstream response so its body can be read again. + * + * A `Response` body can only be consumed once. The loop reads it to decide + * whether the model asked for a server tool, and handing the same object back + * used to leave the route layer — which reads it again to build the answer the + * client actually sees — with a spent body: the second read threw + * "Body already used" and the client got a 500 in place of the real upstream + * status. Draining it here and replaying the bytes in a fresh response keeps + * both reads working and preserves the body verbatim, so an upstream error + * detail that is not valid JSON still reaches the client intact. + * + * `content-length` and `content-encoding` are dropped: the body is re-emitted + * rather than re-encoded, and a stale length would describe bytes the upstream + * compressed before this layer ever saw them. + */ +export const buildServerToolFailureResponse = async ( + response: Response, +): Promise => { + const headers = new Headers(response.headers); + + headers.delete('content-length'); + headers.delete('content-encoding'); + headers.set('content-type', 'application/json'); + + return new Response(await response.text(), { + headers, + status: response.status, + statusText: response.statusText, + }); +}; + +/** + * Parses a buffered upstream body, tolerating a failure that is not JSON. + * + * A successful response must be well-formed — anything else is a bug worth + * surfacing — but a failure status already tells the caller everything it + * needs to know, and its body may legitimately be an HTML error page or a + * bare string. Rejecting on those would turn an ordinary outage into an + * unhandled rejection. + */ +export const parseBufferedPayload = ( + buffered: string, + ok: boolean, +): ChatCompletionPayload => { + try { + return JSON.parse(buffered) as ChatCompletionPayload; + } catch (error) { + if (ok) { + throw error; + } + + return {}; + } +}; + +export const readBufferedChatCompletionPayload = async ( + response: Response, +): Promise => { + // Cloned so the failure path can replay the body verbatim; see + // {@link buildServerToolFailureResponse}. + const buffered = await response.clone().text(); + const payload = parseBufferedPayload(buffered, response.ok); + + if (!response.ok || payload.error) { + const ownMessage = payload.error?.message; + // `extractErrorMessage` digs a nested message out of the payload, so + // `{"error":{"message":"x"}}` reaches the client as "x" rather than as a + // JSON string. The raw body is the fallback: a payload carrying only a + // code has no message to find, and the JSON is still the only record of + // what happened. An empty body says nothing, so it falls all the way + // through to the generic message instead of winning on being non-null. + const detail = buffered.trim(); + + return { + ...payload, + error: { + // The upstream's own explanation — a rate-limit code, a reset + // timestamp — beats the proxy's generic "Upstream CodeBuddy request + // failed", which says only that something failed and leaves the client + // no way to tell what. + // + // `status` travels with the frame so a downstream mapper can name the + // real error type instead of guessing it from the message text. + message: + extractErrorMessage(payload) ?? + ownMessage ?? + (detail || `Upstream request failed with status ${response.status}`), + ...(response.ok ? {} : { status: response.status }), + }, + }; + } + + return payload; +}; diff --git a/lib/server/proxy/server-tool-sse.ts b/lib/server/proxy/server-tool-sse.ts new file mode 100644 index 0000000..413c4bc --- /dev/null +++ b/lib/server/proxy/server-tool-sse.ts @@ -0,0 +1,130 @@ +import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { + STREAM_TEXT_CHUNK_LENGTH, + type ChatCompletionPayload, + type JsonRecord, +} from './server-tool-types'; + +/** + * Replays a buffered completion as chat-completion SSE. Used only after a + * streaming request actually invokes a server-executed tool; ordinary answers + * keep the upstream response untouched. + */ +export const synthesizeChatCompletionStream = ( + payload: ChatCompletionPayload, + fallbackModel: string, +): Response => { + const encoder = new TextEncoder(); + const choice = payload.choices?.[0]; + const message = choice?.message; + const created = + typeof payload.created === 'number' + ? payload.created + : Math.floor(Date.now() / 1000); + const model = + typeof payload.model === 'string' && payload.model + ? payload.model + : fallbackModel; + const id = + typeof payload.id === 'string' && payload.id + ? payload.id + : `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`; + + const enqueue = ( + controller: ReadableStreamDefaultController, + chunk: JsonRecord, + ): void => { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + ...chunk, + created, + id, + model, + object: 'chat.completion.chunk', + })}\n\n`, + ), + ); + }; + + const stream = new ReadableStream({ + start: (controller) => { + enqueue(controller, { + choices: [{ delta: { role: 'assistant' }, index: 0 }], + }); + + const reasoning = message?.reasoning_content ?? message?.reasoning; + + if (reasoning) { + enqueue(controller, { + choices: [{ delta: { reasoning_content: reasoning }, index: 0 }], + }); + } + + const content = + typeof message?.content === 'string' ? message.content : ''; + + for ( + let offset = 0; + offset < content.length; + offset += STREAM_TEXT_CHUNK_LENGTH + ) { + enqueue(controller, { + choices: [ + { + delta: { + content: content.slice( + offset, + offset + STREAM_TEXT_CHUNK_LENGTH, + ), + }, + index: 0, + }, + ], + }); + } + + const passthroughCalls = (message?.tool_calls ?? []).map( + (toolCall, index) => ({ + ...toolCall, + id: toolCall.id ?? `call_${index}`, + index, + type: toolCall.type ?? 'function', + }), + ); + + if (passthroughCalls.length) { + enqueue(controller, { + choices: [{ delta: { tool_calls: passthroughCalls }, index: 0 }], + }); + } + + enqueue(controller, { + choices: [ + { + delta: {}, + finish_reason: + choice?.finish_reason ?? + (passthroughCalls.length ? 'tool_calls' : 'stop'), + index: 0, + }, + ], + }); + + if (payload.usage !== undefined && payload.usage !== null) { + enqueue(controller, { + choices: [], + usage: payload.usage, + }); + } + + controller.enqueue(encodeDoneFrame()); + controller.close(); + }, + }); + + return createSseResponse(stream, { + headers: { 'Access-Control-Allow-Origin': '*' }, + status: 200, + }); +}; diff --git a/lib/server/proxy/server-tool-stream.ts b/lib/server/proxy/server-tool-stream.ts new file mode 100644 index 0000000..a9c1bda --- /dev/null +++ b/lib/server/proxy/server-tool-stream.ts @@ -0,0 +1,233 @@ +import { isLocalServerToolCall } from './server-tool-classify'; +import type { + ChatCompletionMessage, + ChatCompletionToolCall, +} from './server-tool-types'; +import type { WebFetchProvider, WebSearchProvider } from '../search/types'; + +export const mergeStreamingToolName = ( + previous: string, + incoming: string, +): string => { + if (!previous || incoming.startsWith(previous)) return incoming; + if (!incoming || previous.endsWith(incoming)) return previous; + return previous + incoming; +}; + +export const aggregateStreamingToolCalls = ( + deltas: ChatCompletionToolCall[], +): ChatCompletionToolCall[] => { + const calls = new Map< + string, + ChatCompletionToolCall & { + function: { arguments: string; name: string }; + } + >(); + const latestKeyByIndex = new Map(); + + deltas.forEach((delta, position) => { + const indexedKey = + typeof delta.index === 'number' + ? latestKeyByIndex.get(delta.index) + : undefined; + const key = + indexedKey ?? + (delta.id ? `id:${delta.id}` : undefined) ?? + (typeof delta.index === 'number' + ? `index:${delta.index}` + : `position:${position}`); + const current = calls.get(key) ?? { + function: { arguments: '', name: '' }, + index: delta.index, + }; + + current.id = delta.id ?? current.id; + current.index = delta.index ?? current.index; + current.type = delta.type ?? current.type; + current.function.arguments += delta.function?.arguments ?? ''; + current.function.name = mergeStreamingToolName( + current.function.name, + delta.function?.name ?? '', + ); + calls.set(key, current); + + if (typeof delta.index === 'number') { + latestKeyByIndex.set(delta.index, key); + } + }); + + return [...calls.values()]; +}; + +/** + * Result of streaming one upstream response while watching for server-tool + * calls, so a follow-up iteration can decide what happened. + * + * `localCalls` and `remainingCalls` partition the aggregated tool calls the + * way the execution loop needs them; `frames` are the frames that were held + * back because they carried tool-call deltas. + */ +export interface ServerToolProbe { + content: string; + frames: string[]; + localCalls: ChatCompletionToolCall[]; + reasoning: string; + remainingCalls: ChatCompletionToolCall[]; + role: string; + toolCalls: ChatCompletionToolCall[]; + usage: unknown; +} + +export const probeServerToolStream = async ({ + canContinue, + context, + emitRaw, + fetchProvider, + onReader, + ownedNames, + response, + searchProvider, +}: { + canContinue: () => boolean; + context: { + responseCreated: number; + responseId: string; + responseModel: string; + responseObject: string; + role: string; + usage: unknown; + }; + emitRaw: (frame: string) => void; + fetchProvider: WebFetchProvider | null; + ownedNames?: Set; + /** + * Hands the active reader to the caller's cancellation path. Without it a + * disconnect cannot interrupt a read that is already parked: the loop only + * notices the cancellation once upstream produces another chunk, which a + * stalled upstream never does. + */ + onReader?: (reader: ReadableStreamDefaultReader | null) => void; + response: Response; + searchProvider: WebSearchProvider | null; +}): Promise => { + const frames: string[] = []; + const toolCallDeltas: ChatCompletionToolCall[] = []; + const decoder = new TextDecoder(); + const reader = response.body!.getReader(); + onReader?.(reader); + let buffer = ''; + let content = ''; + let reasoning = ''; + + const inspectFrame = (frame: string): void => { + const line = frame + .split(/\r?\n/) + .find((segment) => segment.startsWith('data:')); + + if (!line) { + emitRaw(frame); + return; + } + + const raw = line.slice(5).trim(); + if (!raw) return; + if (raw === '[DONE]') { + frames.push(frame); + return; + } + + try { + const chunk = JSON.parse(raw) as { + choices?: Array<{ + delta?: ChatCompletionMessage & { + tool_calls?: ChatCompletionToolCall[]; + }; + finish_reason?: string | null; + }>; + created?: number; + id?: string; + model?: string; + object?: string; + usage?: unknown; + }; + context.responseId = chunk.id ?? context.responseId; + context.responseModel = chunk.model ?? context.responseModel; + context.responseObject = + chunk.object?.replace(/\.chunk$/, '') ?? context.responseObject; + context.responseCreated = chunk.created ?? context.responseCreated; + context.usage = chunk.usage ?? context.usage; + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + context.role = delta?.role ?? context.role; + content += delta?.content ?? ''; + reasoning += delta?.reasoning_content ?? delta?.reasoning ?? ''; + + // A tool-call frame is held rather than forwarded: if the turn turns out + // to invoke a server tool, the call has to be answered locally instead + // of being handed to the client as an unresolved call. Anything else the + // delta carried — most importantly the text the model wrote before + // deciding to search — still belongs to the visible turn, so it is + // re-emitted without the tool call. + if (delta?.tool_calls?.length) { + toolCallDeltas.push(...delta.tool_calls); + frames.push(frame); + + const visibleDelta = { ...delta }; + delete visibleDelta.tool_calls; + + if (Object.keys(visibleDelta).length) { + const visible = JSON.stringify({ + ...chunk, + choices: [{ ...choice, delta: visibleDelta, finish_reason: null }], + }); + emitRaw(`data: ${visible}`); + } + return; + } + + if (choice?.finish_reason === 'tool_calls') { + frames.push(frame); + return; + } + } catch { + emitRaw(frame); + return; + } + + emitRaw(frame); + }; + + while (true) { + const chunk = await reader.read(); + if (!canContinue()) break; + if (chunk.done) break; + buffer += decoder.decode(chunk.value, { stream: true }); + const split = buffer.split(/\r?\n\r?\n/); + buffer = split.pop() ?? ''; + split.forEach(inspectFrame); + } + + if (buffer.trim()) inspectFrame(buffer); + reader.releaseLock(); + onReader?.(null); + + const toolCalls = aggregateStreamingToolCalls(toolCallDeltas); + const isLocalCall = (toolCall: ChatCompletionToolCall): boolean => + isLocalServerToolCall({ + fetchProvider, + ownedNames, + searchProvider, + toolCall, + }); + + return { + content, + frames, + localCalls: toolCalls.filter(isLocalCall), + reasoning, + remainingCalls: toolCalls.filter((toolCall) => !isLocalCall(toolCall)), + role: context.role, + toolCalls, + usage: context.usage, + }; +}; diff --git a/lib/server/proxy/server-tool-turns.ts b/lib/server/proxy/server-tool-turns.ts new file mode 100644 index 0000000..3d31a19 --- /dev/null +++ b/lib/server/proxy/server-tool-turns.ts @@ -0,0 +1,205 @@ +import { asRecord, readReasoning } from '../shared/content'; +import type { + ChatCompletionMessage, + ChatCompletionPayload, + ChatCompletionToolCall, + JsonRecord, + ServerToolExecution, + ServerToolTurn, +} from './server-tool-types'; + +export const sumUsage = (accumulated: unknown, incoming: unknown): unknown => { + const left = asRecord(accumulated); + const right = asRecord(incoming); + + if (!left) { + return incoming ?? null; + } + + if (!right) { + return accumulated; + } + + const merged: JsonRecord = { ...left }; + + for (const [key, value] of Object.entries(right)) { + const previous = left[key]; + + if (typeof value === 'number' && typeof previous === 'number') { + merged[key] = previous + value; + } else if (value !== undefined) { + merged[key] = value; + } + } + + return merged; +}; + +/** + * Folds completed search results into the assistant text and re-emits the + * outstanding tool calls unchanged, so a turn that mixed search with + * client-side calls stays a valid transcript. The client sees its own calls + * come back as if upstream had returned them directly. + * + * The findings are folded only when the route has no other way to carry them. + * A route that renders them structurally passes + * `findingsAsStructuredBlocks`, and the text is left alone: the results are + * already on the wire as a result block, and a second copy in the prose is + * what the user reads as the model reciting its own search output. + */ +export const buildMixedTurnPayload = ({ + findingsAsStructuredBlocks = false, + message, + payload, + remainingCalls, + searchResults, + usage, +}: { + findingsAsStructuredBlocks?: boolean; + message: ChatCompletionMessage | undefined; + payload: ChatCompletionPayload; + remainingCalls: ChatCompletionToolCall[]; + searchResults: string[]; + usage?: unknown; +}): ChatCompletionPayload => { + const existingText = + typeof message?.content === 'string' && message.content.trim() + ? message.content.trim() + : ''; + const findings = findingsAsStructuredBlocks + ? '' + : searchResults.filter(Boolean).join('\n\n'); + const content = [existingText, findings].filter(Boolean).join('\n\n'); + + return { + ...payload, + ...(usage ? { usage } : {}), + choices: (payload.choices ?? []).map((choice, index) => + index === 0 + ? { + ...choice, + finish_reason: 'tool_calls', + message: { + ...(choice.message ?? {}), + content: content || null, + role: 'assistant', + tool_calls: remainingCalls, + }, + } + : choice, + ), + }; +}; + +/** + * Pairs each hop's reasoning and text with the calls that hop made. + * + * The three arrays are index-aligned — entry N is hop N — so zipping them back + * together is what restores the grouping a joined string cannot express. `texts` + * and `reasonings` are one entry longer than `executions`, because the closing + * hop answers instead of calling another tool. + */ +export const buildIntermediateTurns = ({ + executions, + reasonings, + texts, +}: { + executions: ServerToolExecution[][]; + reasonings: string[]; + texts: string[]; +}): ServerToolTurn[] => + Array.from( + { length: Math.max(reasonings.length, texts.length) }, + (_, index) => ({ + // Only `executions` can run short: the closing hop answers without + // calling anything, so it has an entry in the prose arrays but none here. + // The three arrays stay aligned because every hop appends to all of them. + executions: executions[index] ?? [], + reasoning: reasonings[index], + text: texts[index], + }), + ); + +/** + * Folds the text a multi-hop turn produced before its later server-tool calls + * into the payload the client receives. + * + * Only the last iteration's message is in `payload`, but a turn that searched + * more than once spoke before each search, and that text is part of the turn: + * dropping it hides the model's reasoning from the user and leaves the + * client's transcript out of step with what the model actually said. + * + * The same hops are also re-grouped into the `turns` half of the result, + * because the folded strings cannot express where one hop ends and the next + * begins. That half travels beside the response rather than inside it: the + * payload is what an OpenAI-protocol client receives, and `turns` is not part + * of that protocol, so it is handed over out of band like `executions`. + * + * Shared with the image-generation loop, which has the same shape: a local + * tool call is replayed with its result appended, so only the final hop's + * message would otherwise survive. + */ +export const withIntermediateTurns = ({ + executions, + payload, + reasonings, + texts, +}: { + executions: ServerToolExecution[][]; + payload: ChatCompletionPayload; + reasonings: string[]; + texts: string[]; +}): { payload: ChatCompletionPayload; turns: ServerToolTurn[] } => { + const extraText = texts.filter(Boolean).join('\n\n'); + const extraReasoning = reasonings.filter(Boolean).join('\n\n'); + const [first, ...rest] = payload.choices ?? []; + + if (!first) { + return { payload, turns: [] }; + } + + const message = first.message ?? {}; + const existingText = + typeof message.content === 'string' ? message.content : ''; + + // Nothing from the earlier hops and nothing to fold in — but the hops may + // still have run tools, which is exactly the case a caller consuming `turns` + // needs: a hop that called a tool without speaking first is still a hop. + if (!extraText && !extraReasoning && !executions.length) { + return { payload, turns: [] }; + } + + const content = [extraText, existingText].filter(Boolean).join('\n\n'); + const reasoning = [extraReasoning, readReasoning(message)] + .filter(Boolean) + .join('\n\n'); + + return { + payload: { + ...payload, + choices: [ + { + ...first, + message: { + ...message, + content, + ...(reasoning ? { reasoning_content: reasoning } : {}), + }, + }, + ...rest, + ], + }, + // Per-hop grouping for renderers that can express it. The joined strings + // above stay as the OpenAI-shaped view; a client that builds Anthropic + // content blocks needs to know where one hop's reasoning ends and the next + // begins, which a joined string has already lost. The closing hop is the + // model's final answer, so it carries no further calls. + turns: buildIntermediateTurns({ + executions, + reasonings: [...reasonings, readReasoning(message)], + texts: [...texts, existingText], + }), + }; +}; + +export { readReasoning }; diff --git a/lib/server/proxy/server-tool-types.ts b/lib/server/proxy/server-tool-types.ts new file mode 100644 index 0000000..3ca1e50 --- /dev/null +++ b/lib/server/proxy/server-tool-types.ts @@ -0,0 +1,129 @@ +import type { + WebFetchQuery, + WebFetchResponse, + WebSearchResponse, +} from '../search/types'; +import type { ChatRequestBody } from './codebuddy'; + +export const MAX_SEARCH_ITERATIONS = 5; +export const STREAM_TEXT_CHUNK_LENGTH = 1024; + +export type JsonRecord = Record; + +export interface ChatCompletionToolCall { + id?: string; + index?: number; + type?: string; + function?: { + arguments?: string; + name?: string; + }; +} + +export interface ChatCompletionMessage { + content?: string | null; + reasoning?: string; + reasoning_content?: string; + role?: string; + tool_calls?: ChatCompletionToolCall[]; +} + +export interface ChatCompletionPayload { + choices?: Array<{ + finish_reason?: string | null; + index?: number; + message?: ChatCompletionMessage; + }>; + created?: number; + /** + * `status` is the upstream HTTP status, carried so a downstream mapper can + * name the real error type instead of guessing it from the message text. It + * is absent for a payload that already reported an error of its own. + */ + error?: { message?: string; status?: number }; + id?: string; + model?: string; + object?: string; + usage?: unknown; +} + +/** + * Result of one server-tool pass. + * + * `response` is null when no tool could be executed: the request still has to + * be sent, but with the server-tool declarations already stripped, so the + * caller falls through to its ordinary upstream path. + */ +export interface ServerToolLoopResult { + body: ChatRequestBody; + executions: ServerToolExecution[]; + response: Response | null; + /** + * One entry per server-tool hop, in the order the model produced them. + * + * Carries the grouping `message.content` / `reasoning_content` cannot: a + * multi-hop turn joins every hop into one string per kind, which loses where + * one hop's reasoning ends and the next begins. Travels beside the response + * rather than inside it, because it is not part of the OpenAI protocol — a + * block renderer reads it off the response through `getServerToolTurns`. + * Empty when no hop ran. + */ + turns: ServerToolTurn[]; +} + +export type ServerToolInvocation = + | { + id: string; + input: { query: string }; + type: 'web_search'; + } + | { + id: string; + input: WebFetchQuery; + type: 'web_fetch'; + }; + +export type ServerToolExecution = + | (Extract & { + result: WebSearchResponse; + }) + | (Extract & { + result: WebFetchResponse; + }); + +/** + * One server-tool hop as the model produced it. + * + * `reasoning` and `text` are what the model wrote before the calls in + * `executions`; both are empty when it called tools without speaking first. The + * last hop of a turn usually has no executions, because the model answered + * instead of reaching for another tool. + */ +export interface ServerToolTurn { + executions: ServerToolExecution[]; + reasoning: string; + text: string; +} + +export interface ServerToolCallbacks { + emitStreamEvents?: boolean; + /** + * Set by routes that render a server tool's findings structurally — + * Anthropic's `web_search_tool_result` block — instead of as prose. Those + * routes must not also fold the same findings into the assistant text, or + * the user sees the results twice: once as a result block and once as if + * the model had written them. + */ + findingsAsStructuredBlocks?: boolean; + onCall?: (invocation: ServerToolInvocation) => void; + onResult?: (execution: ServerToolExecution) => void; +} + +export const SERVER_TOOL_STREAM_EVENT_KEY = 'x-codebuddy2api-server-tool'; + +export type ServerToolStreamEvent = + | { invocation: ServerToolInvocation; phase: 'call' } + | { execution: ServerToolExecution; phase: 'result' }; + +export type ServerToolUpstreamMode = + 'buffer' | 'detect-both' | 'detect-fetch' | 'detect-search' | 'stream'; diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index 35e39c3..93a4360 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -4,36 +4,52 @@ import { isWebFetchEnabled, isWebSearchEnabled, } from '../domain/config'; -import { - resolveFetchProvider, - resolveSearchProvider, - runWebFetchResult, - runWebSearchResult, -} from '../search'; -import { asRecord, readReasoning } from '../shared/content'; -import { extractErrorMessage } from '../shared/http'; -import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { resolveFetchProvider, resolveSearchProvider } from '../search'; +import { normalizeSearchBackend } from '../search/tool'; +import type { WebFetchProvider, WebSearchProvider } from '../search/types'; +import { encodeDoneFrame } from '../shared/sse'; import type { ChatRequestBody } from './codebuddy'; import { - buildWebFetchToolDefinition, - buildWebSearchToolDefinition, - isMarkedServerTool, - normalizeSearchBackend, - normalizeToolName, - stripServerToolMarker, - WEB_FETCH_TOOL_NAME, - WEB_FETCH_TOOL_TYPE_PREFIX, - WEB_SEARCH_TOOL_NAME, - WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; -import type { - WebFetchProvider, - WebFetchQuery, - WebFetchResponse, - WebSearchProvider, - WebSearchResponse, -} from '../search/types'; + isLocalServerToolCall, + isWebFetchTool, + isWebSearchTool, + replaceServerTools, +} from './server-tool-classify'; +import { + buildServerToolInvocation, + executeServerToolInvocations, +} from './server-tool-execution'; +import { + buildServerToolFailureResponse, + parseBufferedPayload, + readBufferedChatCompletionPayload, +} from './server-tool-payload'; +import { synthesizeChatCompletionStream } from './server-tool-sse'; +import { + type ServerToolProbe, + probeServerToolStream, +} from './server-tool-stream'; +import { + buildMixedTurnPayload, + readReasoning, + sumUsage, + withIntermediateTurns, +} from './server-tool-turns'; +import { + MAX_SEARCH_ITERATIONS, + SERVER_TOOL_STREAM_EVENT_KEY, + type ChatCompletionMessage, + type ChatCompletionPayload, + type ChatCompletionToolCall, + type JsonRecord, + type ServerToolCallbacks, + type ServerToolExecution, + type ServerToolLoopResult, + type ServerToolStreamEvent, + type ServerToolTurn, + type ServerToolUpstreamMode, +} from './server-tool-types'; /** * Server-side web search for upstreams that do not implement it. @@ -51,1102 +67,6 @@ import type { * is buffered because its arguments are only complete once that response ends. */ -const MAX_SEARCH_ITERATIONS = 5; -const STREAM_TEXT_CHUNK_LENGTH = 1024; - -type JsonRecord = Record; - -export interface ChatCompletionToolCall { - id?: string; - index?: number; - type?: string; - function?: { - arguments?: string; - name?: string; - }; -} - -export interface ChatCompletionMessage { - content?: string | null; - reasoning?: string; - reasoning_content?: string; - role?: string; - tool_calls?: ChatCompletionToolCall[]; -} - -export interface ChatCompletionPayload { - choices?: Array<{ - finish_reason?: string | null; - index?: number; - message?: ChatCompletionMessage; - }>; - created?: number; - /** - * `status` is the upstream HTTP status, carried so a downstream mapper can - * name the real error type instead of guessing it from the message text. It - * is absent for a payload that already reported an error of its own. - */ - error?: { message?: string; status?: number }; - id?: string; - model?: string; - object?: string; - usage?: unknown; -} - -/** - * Rebuilds a failed upstream response so its body can be read again. - * - * A `Response` body can only be consumed once. The loop reads it to decide - * whether the model asked for a server tool, and handing the same object back - * used to leave the route layer — which reads it again to build the answer the - * client actually sees — with a spent body: the second read threw - * "Body already used" and the client got a 500 in place of the real upstream - * status. Draining it here and replaying the bytes in a fresh response keeps - * both reads working and preserves the body verbatim, so an upstream error - * detail that is not valid JSON still reaches the client intact. - * - * `content-length` and `content-encoding` are dropped: the body is re-emitted - * rather than re-encoded, and a stale length would describe bytes the upstream - * compressed before this layer ever saw them. - */ -const buildServerToolFailureResponse = async ( - response: Response, -): Promise => { - const headers = new Headers(response.headers); - - headers.delete('content-length'); - headers.delete('content-encoding'); - headers.set('content-type', 'application/json'); - - return new Response(await response.text(), { - headers, - status: response.status, - statusText: response.statusText, - }); -}; - -/** - * Parses a buffered upstream body, tolerating a failure that is not JSON. - * - * A successful response must be well-formed — anything else is a bug worth - * surfacing — but a failure status already tells the caller everything it - * needs to know, and its body may legitimately be an HTML error page or a - * bare string. Rejecting on those would turn an ordinary outage into an - * unhandled rejection. - */ -const parseBufferedPayload = ( - buffered: string, - ok: boolean, -): ChatCompletionPayload => { - try { - return JSON.parse(buffered) as ChatCompletionPayload; - } catch (error) { - if (ok) { - throw error; - } - - return {}; - } -}; - -const readBufferedChatCompletionPayload = async ( - response: Response, -): Promise => { - // Cloned so the failure path can replay the body verbatim; see - // {@link buildServerToolFailureResponse}. - const buffered = await response.clone().text(); - const payload = parseBufferedPayload(buffered, response.ok); - - if (!response.ok || payload.error) { - const ownMessage = payload.error?.message; - // `extractErrorMessage` digs a nested message out of the payload, so - // `{"error":{"message":"x"}}` reaches the client as "x" rather than as a - // JSON string. The raw body is the fallback: a payload carrying only a - // code has no message to find, and the JSON is still the only record of - // what happened. An empty body says nothing, so it falls all the way - // through to the generic message instead of winning on being non-null. - const detail = buffered.trim(); - - return { - ...payload, - error: { - // The upstream's own explanation — a rate-limit code, a reset - // timestamp — beats the proxy's generic "Upstream CodeBuddy request - // failed", which says only that something failed and leaves the client - // no way to tell what. - // - // `status` travels with the frame so a downstream mapper can name the - // real error type instead of guessing it from the message text. - message: - extractErrorMessage(payload) ?? - ownMessage ?? - (detail || `Upstream request failed with status ${response.status}`), - ...(response.ok ? {} : { status: response.status }), - }, - }; - } - - return payload; -}; - -/** - * Recognises one server-tool declaration. - * - * Anthropic sends dated server-tool *types* (`web_search_20260209`, - * `web_fetch_20250910`), Responses sends `web_search_preview`, and a client may - * also declare a plain function tool with the bare name for its own purposes. - * All three shapes have to match, because the tool has to be swapped for a - * function upstream can actually call regardless of how it arrived. - * - * Returns two independent answers. `matches` says the declaration is one this - * proxy can serve; `serverDeclared` says it arrived as a provider-executed - * server tool rather than as the client's own function. The difference decides - * what happens when the tool cannot be executed: a server-tool declaration is - * dropped, because upstream has no idea what to do with it, whereas the - * client's own function is left exactly as sent — the client is the one that - * resolves it, and deleting it would silently remove a capability the client - * asked for. - */ -const classifyServerTool = ( - tool: unknown, - name: string, - prefix: string, -): { matches: boolean; serverDeclared: boolean } => { - const record = asRecord(tool); - - if (!record) { - return { matches: false, serverDeclared: false }; - } - - const type = typeof record.type === 'string' ? record.type : ''; - - // A dedicated server-tool type (`web_search_20260209`, `web_fetch_20250910`, - // `web_search_preview`) is unambiguous: only a provider-executed tool is - // declared that way. The trailing date is part of the version, not the name, - // so the prefix is matched in canonical form — `WebFetch_20250910` arrives - // from upstream as readily as its snake_case spelling. - if (normalizeToolName(type).startsWith(normalizeToolName(prefix))) { - return { matches: true, serverDeclared: true }; - } - - const fn = asRecord(record.function); - const isBareName = - (typeof fn?.name === 'string' && - normalizeToolName(fn.name) === normalizeToolName(name)) || - (typeof record.name === 'string' && - normalizeToolName(record.name).startsWith(normalizeToolName(prefix))); - - // A Responses translation has already flattened the declaration into a plain - // function, so the type is gone by now; its marker is the only surviving - // evidence that the client asked for a provider-executed tool. - return { - matches: isBareName, - serverDeclared: isBareName && isMarkedServerTool(tool), - }; -}; - -const isWebSearchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) - .matches; - -const isWebFetchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) - .matches; - -const isServerDeclaredSearchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX) - .serverDeclared; - -const isServerDeclaredFetchTool = (tool: unknown): boolean => - classifyServerTool(tool, WEB_FETCH_TOOL_NAME, WEB_FETCH_TOOL_TYPE_PREFIX) - .serverDeclared; - -/** - * Whether `toolCall` is a call the proxy is meant to execute. - * - * Matched in canonical form because the name comes back from the model, which - * is under no obligation to repeat the spelling it was given: upstream echoes - * `web_fetch` as `WebFetch` often enough to matter here. A miss is not a - * fallback to the client — the call leaves the loop as an unanswered - * client-owned tool, so the fetch silently never happens. - */ -const isWebSearchToolCall = (toolCall: ChatCompletionToolCall): boolean => { - return ( - typeof toolCall.function?.name === 'string' && - normalizeToolName(toolCall.function.name) === - normalizeToolName(WEB_SEARCH_TOOL_NAME) - ); -}; - -const isWebFetchToolCall = (toolCall: ChatCompletionToolCall): boolean => { - return ( - typeof toolCall.function?.name === 'string' && - normalizeToolName(toolCall.function.name) === - normalizeToolName(WEB_FETCH_TOOL_NAME) - ); -}; - -/** - * Swaps locally executed server-tool declarations for functions upstream can - * call. Passthrough tools keep their upstream representation. - * - * Returns `null` when no web tool is present. `executes` distinguishes a local - * backend from passthrough: the latter still strips the internal provenance - * marker, but never starts the server loop or buffers a stream. - */ -const replaceServerTools = ({ - fetchEnabled, - fetchProvider, - searchEnabled, - searchPassthrough, - searchProvider, - tools, -}: { - fetchEnabled: boolean; - fetchProvider: WebFetchProvider | null; - searchEnabled: boolean; - searchPassthrough: boolean; - searchProvider: WebSearchProvider | null; - tools: unknown; -}): { - executes: boolean; - /** Canonical names the proxy took over, so call classification can tell its own calls from a client's. */ - ownedNames: Set; - tools: unknown[]; -} | null => { - if (!Array.isArray(tools) || !tools.length) { - return null; - } - - let matched = false; - let executes = false; - // Names the proxy is executing itself. A client may declare its own tool - // under the same name, and the loop must not answer those calls: matching - // the name is not enough to own it. - const ownedNames = new Set(); - - const rewritten = tools.flatMap((tool): unknown[] => { - if (isWebSearchTool(tool)) { - if (searchEnabled && searchProvider) { - matched = true; - executes = true; - ownedNames.add(normalizeToolName(WEB_SEARCH_TOOL_NAME)); - - return [{ type: 'function', function: buildWebSearchToolDefinition() }]; - } - - if (!isServerDeclaredSearchTool(tool)) { - return [tool]; - } - - matched = true; - return searchPassthrough ? [stripServerToolMarker(tool)] : []; - } - - if (isWebFetchTool(tool)) { - // A client-owned function of the same name wins over the backend, exactly - // as it does for search. The backend setting chooses who runs the *proxy's* - // tool; it is not a licence to take over a tool the client declared and - // resolves itself. Without this, a client that ships its own `web_fetch` - // loses it the moment a deployment picks a backend. - if (!isServerDeclaredFetchTool(tool)) { - return [tool]; - } - - if (fetchEnabled && fetchProvider) { - matched = true; - executes = true; - ownedNames.add(normalizeToolName(WEB_FETCH_TOOL_NAME)); - - return [{ type: 'function', function: buildWebFetchToolDefinition() }]; - } - - matched = true; - return [stripServerToolMarker(tool)]; - } - - // The marker is internal to this proxy, so it never reaches upstream. - return [stripServerToolMarker(tool)]; - }); - - return matched ? { executes, ownedNames, tools: rewritten } : null; -}; - -/** - * Reads one string field out of a tool-call argument object. - * - * Backends expect a single string, but models emit `query`, `q`, - * `search_query`, or an Anthropic-style `{query: {q: ...}}` nested object, so - * any string-ish value is accepted rather than failing the call. - */ -const extractStringArgument = ({ - keys, - rawArguments, - required, -}: { - keys: string[]; - rawArguments: string | undefined; - required: boolean; -}): string => { - if (!rawArguments) { - return ''; - } - - try { - const parsed = JSON.parse(rawArguments) as unknown; - - // Some clients send a bare JSON string rather than an object. - if (typeof parsed === 'string') { - return parsed.trim(); - } - - const record = asRecord(parsed); - - if (!record) { - return ''; - } - - for (const key of keys) { - const value = record[key]; - - if (typeof value === 'string' && value.trim()) { - return value.trim(); - } - - // Anthropic-style arguments nest the value one level deeper. - const nested = asRecord(value); - - if (nested) { - for (const nestedKey of keys) { - const nestedValue = nested[nestedKey]; - - if (typeof nestedValue === 'string' && nestedValue.trim()) { - return nestedValue.trim(); - } - } - } - } - - if (required) { - // Fall back to whichever field holds the first non-empty string, so an - // unexpected argument shape still yields a usable value. Only safe when - // every field means the same thing, which is true for a single-string - // search query but not for a fetch's url plus prompt. - const firstString = Object.values(record).find( - (value): value is string => - typeof value === 'string' && value.trim().length > 0, - ); - - return firstString?.trim() ?? ''; - } - - return ''; - } catch { - // Malformed JSON: treat the raw text as the value so the call still runs. - return rawArguments.trim(); - } -}; - -const extractSearchQuery = (rawArguments: string | undefined): string => - extractStringArgument({ - keys: ['query', 'q', 'search_query', 'text'], - rawArguments, - required: true, - }); - -/** - * Builds the `web_fetch` arguments. - * - * A missing URL is reported to the model rather than thrown: the model sent the - * call, so telling it the argument was missing lets it retry correctly, whereas - * an exception would surface as an opaque tool failure. - */ -const extractFetchQuery = (rawArguments: string | undefined): WebFetchQuery => { - const url = extractStringArgument({ - keys: ['url', 'uri', 'link'], - rawArguments, - required: false, - }); - const prompt = extractStringArgument({ - keys: ['prompt', 'question', 'goal'], - rawArguments, - required: false, - }); - - return { ...(prompt ? { prompt } : {}), url }; -}; - -const sumUsage = (accumulated: unknown, incoming: unknown): unknown => { - const left = asRecord(accumulated); - const right = asRecord(incoming); - - if (!left) { - return incoming ?? null; - } - - if (!right) { - return accumulated; - } - - const merged: JsonRecord = { ...left }; - - for (const [key, value] of Object.entries(right)) { - const previous = left[key]; - - if (typeof value === 'number' && typeof previous === 'number') { - merged[key] = previous + value; - } else if (value !== undefined) { - merged[key] = value; - } - } - - return merged; -}; - -/** - * Folds completed search results into the assistant text and re-emits the - * outstanding tool calls unchanged, so a turn that mixed search with - * client-side calls stays a valid transcript. The client sees its own calls - * come back as if upstream had returned them directly. - * - * The findings are folded only when the route has no other way to carry them. - * A route that renders them structurally passes - * `findingsAsStructuredBlocks`, and the text is left alone: the results are - * already on the wire as a result block, and a second copy in the prose is - * what the user reads as the model reciting its own search output. - */ -const buildMixedTurnPayload = ({ - findingsAsStructuredBlocks = false, - message, - payload, - remainingCalls, - searchResults, - usage, -}: { - findingsAsStructuredBlocks?: boolean; - message: ChatCompletionMessage | undefined; - payload: ChatCompletionPayload; - remainingCalls: ChatCompletionToolCall[]; - searchResults: string[]; - usage?: unknown; -}): ChatCompletionPayload => { - const existingText = - typeof message?.content === 'string' && message.content.trim() - ? message.content.trim() - : ''; - const findings = findingsAsStructuredBlocks - ? '' - : searchResults.filter(Boolean).join('\n\n'); - const content = [existingText, findings].filter(Boolean).join('\n\n'); - - return { - ...payload, - ...(usage ? { usage } : {}), - choices: (payload.choices ?? []).map((choice, index) => - index === 0 - ? { - ...choice, - finish_reason: 'tool_calls', - message: { - ...(choice.message ?? {}), - content: content || null, - role: 'assistant', - tool_calls: remainingCalls, - }, - } - : choice, - ), - }; -}; - -/** - * Pairs each hop's reasoning and text with the calls that hop made. - * - * The three arrays are index-aligned — entry N is hop N — so zipping them back - * together is what restores the grouping a joined string cannot express. `texts` - * and `reasonings` are one entry longer than `executions`, because the closing - * hop answers instead of calling another tool. - */ -const buildIntermediateTurns = ({ - executions, - reasonings, - texts, -}: { - executions: ServerToolExecution[][]; - reasonings: string[]; - texts: string[]; -}): ServerToolTurn[] => - Array.from( - { length: Math.max(reasonings.length, texts.length) }, - (_, index) => ({ - // Only `executions` can run short: the closing hop answers without - // calling anything, so it has an entry in the prose arrays but none here. - // The three arrays stay aligned because every hop appends to all of them. - executions: executions[index] ?? [], - reasoning: reasonings[index], - text: texts[index], - }), - ); - -/** - * Folds the text a multi-hop turn produced before its later server-tool calls - * into the payload the client receives. - * - * Only the last iteration's message is in `payload`, but a turn that searched - * more than once spoke before each search, and that text is part of the turn: - * dropping it hides the model's reasoning from the user and leaves the - * client's transcript out of step with what the model actually said. - * - * The same hops are also re-grouped into the `turns` half of the result, - * because the folded strings cannot express where one hop ends and the next - * begins. That half travels beside the response rather than inside it: the - * payload is what an OpenAI-protocol client receives, and `turns` is not part - * of that protocol, so it is handed over out of band like `executions`. - * - * Shared with the image-generation loop, which has the same shape: a local - * tool call is replayed with its result appended, so only the final hop's - * message would otherwise survive. - */ -export const withIntermediateTurns = ({ - executions, - payload, - reasonings, - texts, -}: { - executions: ServerToolExecution[][]; - payload: ChatCompletionPayload; - reasonings: string[]; - texts: string[]; -}): { payload: ChatCompletionPayload; turns: ServerToolTurn[] } => { - const extraText = texts.filter(Boolean).join('\n\n'); - const extraReasoning = reasonings.filter(Boolean).join('\n\n'); - const [first, ...rest] = payload.choices ?? []; - - if (!first) { - return { payload, turns: [] }; - } - - const message = first.message ?? {}; - const existingText = - typeof message.content === 'string' ? message.content : ''; - - // Nothing from the earlier hops and nothing to fold in — but the hops may - // still have run tools, which is exactly the case a caller consuming `turns` - // needs: a hop that called a tool without speaking first is still a hop. - if (!extraText && !extraReasoning && !executions.length) { - return { payload, turns: [] }; - } - - const content = [extraText, existingText].filter(Boolean).join('\n\n'); - const reasoning = [extraReasoning, readReasoning(message)] - .filter(Boolean) - .join('\n\n'); - - return { - payload: { - ...payload, - choices: [ - { - ...first, - message: { - ...message, - content, - ...(reasoning ? { reasoning_content: reasoning } : {}), - }, - }, - ...rest, - ], - }, - // Per-hop grouping for renderers that can express it. The joined strings - // above stay as the OpenAI-shaped view; a client that builds Anthropic - // content blocks needs to know where one hop's reasoning ends and the next - // begins, which a joined string has already lost. The closing hop is the - // model's final answer, so it carries no further calls. - turns: buildIntermediateTurns({ - executions, - reasonings: [...reasonings, readReasoning(message)], - texts: [...texts, existingText], - }), - }; -}; - -/** - * Result of one server-tool pass. - * - * `response` is null when no tool could be executed: the request still has to - * be sent, but with the server-tool declarations already stripped, so the - * caller falls through to its ordinary upstream path. - */ -export interface ServerToolLoopResult { - body: ChatRequestBody; - executions: ServerToolExecution[]; - response: Response | null; - /** - * One entry per server-tool hop, in the order the model produced them. - * - * Carries the grouping `message.content` / `reasoning_content` cannot: a - * multi-hop turn joins every hop into one string per kind, which loses where - * one hop's reasoning ends and the next begins. Travels beside the response - * rather than inside it, because it is not part of the OpenAI protocol — a - * block renderer reads it off the response through `getServerToolTurns`. - * Empty when no hop ran. - */ - turns: ServerToolTurn[]; -} - -export type ServerToolInvocation = - | { - id: string; - input: { query: string }; - type: 'web_search'; - } - | { - id: string; - input: WebFetchQuery; - type: 'web_fetch'; - }; - -export type ServerToolExecution = - | (Extract & { - result: WebSearchResponse; - }) - | (Extract & { - result: WebFetchResponse; - }); - -/** - * One server-tool hop as the model produced it. - * - * `reasoning` and `text` are what the model wrote before the calls in - * `executions`; both are empty when it called tools without speaking first. The - * last hop of a turn usually has no executions, because the model answered - * instead of reaching for another tool. - */ -export interface ServerToolTurn { - executions: ServerToolExecution[]; - reasoning: string; - text: string; -} - -export interface ServerToolCallbacks { - emitStreamEvents?: boolean; - /** - * Set by routes that render a server tool's findings structurally — - * Anthropic's `web_search_tool_result` block — instead of as prose. Those - * routes must not also fold the same findings into the assistant text, or - * the user sees the results twice: once as a result block and once as if - * the model had written them. - */ - findingsAsStructuredBlocks?: boolean; - onCall?: (invocation: ServerToolInvocation) => void; - onResult?: (execution: ServerToolExecution) => void; -} - -const SERVER_TOOL_STREAM_EVENT_KEY = 'x-codebuddy2api-server-tool'; - -export type ServerToolStreamEvent = - | { invocation: ServerToolInvocation; phase: 'call' } - | { execution: ServerToolExecution; phase: 'result' }; - -export const getServerToolStreamEvent = ( - value: unknown, -): ServerToolStreamEvent | null => { - const record = asRecord(value); - const event = asRecord(record?.[SERVER_TOOL_STREAM_EVENT_KEY]); - - if (event?.phase === 'call' && event.invocation) { - return { - invocation: event.invocation as ServerToolInvocation, - phase: 'call', - }; - } - - if (event?.phase === 'result' && event.execution) { - return { - execution: event.execution as ServerToolExecution, - phase: 'result', - }; - } - - return null; -}; - -const serverToolExecutions = new WeakMap(); - -export const attachServerToolExecutions = ( - response: Response, - executions: ServerToolExecution[], -): Response => { - if (executions.length) { - serverToolExecutions.set(response, executions); - } - - return response; -}; - -export const getServerToolExecutions = ( - response: Response, -): ServerToolExecution[] => serverToolExecutions.get(response) ?? []; - -/** - * Per-hop grouping, kept off the wire for the same reason `executions` is: it - * is not part of the OpenAI protocol, so a `/v1/chat/completions` client must - * not see it — a strict validator can reject the extra field, and the tool - * data would otherwise be sent twice. - */ -const serverToolTurns = new WeakMap(); - -export const attachServerToolTurns = ( - response: Response, - turns: ServerToolTurn[], -): Response => { - if (turns.length) { - serverToolTurns.set(response, turns); - } - - return response; -}; - -export const getServerToolTurns = ( - response: Response, -): ServerToolTurn[] | undefined => serverToolTurns.get(response); - -export type ServerToolUpstreamMode = - 'buffer' | 'detect-both' | 'detect-fetch' | 'detect-search' | 'stream'; - -const mergeStreamingToolName = (previous: string, incoming: string): string => { - if (!previous || incoming.startsWith(previous)) return incoming; - if (!incoming || previous.endsWith(incoming)) return previous; - return previous + incoming; -}; - -const aggregateStreamingToolCalls = ( - deltas: ChatCompletionToolCall[], -): ChatCompletionToolCall[] => { - const calls = new Map< - string, - ChatCompletionToolCall & { - function: { arguments: string; name: string }; - } - >(); - const latestKeyByIndex = new Map(); - - deltas.forEach((delta, position) => { - const indexedKey = - typeof delta.index === 'number' - ? latestKeyByIndex.get(delta.index) - : undefined; - const key = - indexedKey ?? - (delta.id ? `id:${delta.id}` : undefined) ?? - (typeof delta.index === 'number' - ? `index:${delta.index}` - : `position:${position}`); - const current = calls.get(key) ?? { - function: { arguments: '', name: '' }, - index: delta.index, - }; - - current.id = delta.id ?? current.id; - current.index = delta.index ?? current.index; - current.type = delta.type ?? current.type; - current.function.arguments += delta.function?.arguments ?? ''; - current.function.name = mergeStreamingToolName( - current.function.name, - delta.function?.name ?? '', - ); - calls.set(key, current); - - if (typeof delta.index === 'number') { - latestKeyByIndex.set(delta.index, key); - } - }); - - return [...calls.values()]; -}; - -const buildServerToolInvocation = ( - toolCall: ChatCompletionToolCall, - iteration: number, - index: number, -): ServerToolInvocation => - isWebFetchToolCall(toolCall) - ? { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: extractFetchQuery(toolCall.function?.arguments), - type: 'web_fetch', - } - : { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: { query: extractSearchQuery(toolCall.function?.arguments) }, - type: 'web_search', - }; - -const executeServerToolInvocations = async ({ - callbacks, - fetchProvider, - invocations, - searchProvider, -}: { - callbacks?: ServerToolCallbacks; - fetchProvider: WebFetchProvider | null; - invocations: ServerToolInvocation[]; - searchProvider: WebSearchProvider | null; -}): Promise< - Array<{ - content: string; - execution: ServerToolExecution; - tool_call_id: string; - }> -> => { - invocations.forEach((invocation) => callbacks?.onCall?.(invocation)); - - return await Promise.all( - invocations.map(async (invocation) => { - if (invocation.type === 'web_fetch') { - const result = await runWebFetchResult({ - provider: fetchProvider, - query: invocation.input, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - } - - const result = await runWebSearchResult({ - provider: searchProvider, - query: invocation.input.query, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - }), - ); -}; - -/** - * Result of streaming one upstream response while watching for server-tool - * calls, so a follow-up iteration can decide what happened. - * - * `localCalls` and `remainingCalls` partition the aggregated tool calls the - * way the execution loop needs them; `frames` are the frames that were held - * back because they carried tool-call deltas. - */ -interface ServerToolProbe { - content: string; - frames: string[]; - localCalls: ChatCompletionToolCall[]; - reasoning: string; - remainingCalls: ChatCompletionToolCall[]; - role: string; - toolCalls: ChatCompletionToolCall[]; - usage: unknown; -} - -/** - * Whether the proxy is the one meant to answer this call. - * - * A call without an available backend is not a fallback to the client — it - * leaves the loop as an unanswered client-owned tool — but it must not be - * counted as locally executable either. - */ -const isLocalServerToolCall = ({ - fetchProvider, - ownedNames, - toolCall, - searchProvider, -}: { - fetchProvider: WebFetchProvider | null; - /** - * Canonical names the proxy took over. Without it a client's own tool that - * happens to share a name — `web_fetch`, which is not a server tool in the - * Responses API — gets executed by the loop instead of handed back. - */ - ownedNames?: Set; - toolCall: ChatCompletionToolCall; - searchProvider: WebSearchProvider | null; -}): boolean => - (Boolean(searchProvider) && - isWebSearchToolCall(toolCall) && - isOwned(ownedNames, WEB_SEARCH_TOOL_NAME)) || - (Boolean(fetchProvider) && - isWebFetchToolCall(toolCall) && - isOwned(ownedNames, WEB_FETCH_TOOL_NAME)); - -/** - * Whether the proxy owns calls to `name`. - * - * `undefined` means the caller predates ownership tracking; those callers only - * ever run the proxy's own declarations, so they are unaffected by client tools - * of the same name. - */ -const isOwned = (ownedNames: Set | undefined, name: string): boolean => - !ownedNames || ownedNames.has(normalizeToolName(name)); - -const probeServerToolStream = async ({ - canContinue, - context, - emitRaw, - fetchProvider, - onReader, - ownedNames, - response, - searchProvider, -}: { - canContinue: () => boolean; - context: { - responseCreated: number; - responseId: string; - responseModel: string; - responseObject: string; - role: string; - usage: unknown; - }; - emitRaw: (frame: string) => void; - fetchProvider: WebFetchProvider | null; - ownedNames?: Set; - /** - * Hands the active reader to the caller's cancellation path. Without it a - * disconnect cannot interrupt a read that is already parked: the loop only - * notices the cancellation once upstream produces another chunk, which a - * stalled upstream never does. - */ - onReader?: (reader: ReadableStreamDefaultReader | null) => void; - response: Response; - searchProvider: WebSearchProvider | null; -}): Promise => { - const frames: string[] = []; - const toolCallDeltas: ChatCompletionToolCall[] = []; - const decoder = new TextDecoder(); - const reader = response.body!.getReader(); - onReader?.(reader); - let buffer = ''; - let content = ''; - let reasoning = ''; - - const inspectFrame = (frame: string): void => { - const line = frame - .split(/\r?\n/) - .find((segment) => segment.startsWith('data:')); - - if (!line) { - emitRaw(frame); - return; - } - - const raw = line.slice(5).trim(); - if (!raw) return; - if (raw === '[DONE]') { - frames.push(frame); - return; - } - - try { - const chunk = JSON.parse(raw) as { - choices?: Array<{ - delta?: ChatCompletionMessage & { - tool_calls?: ChatCompletionToolCall[]; - }; - finish_reason?: string | null; - }>; - created?: number; - id?: string; - model?: string; - object?: string; - usage?: unknown; - }; - context.responseId = chunk.id ?? context.responseId; - context.responseModel = chunk.model ?? context.responseModel; - context.responseObject = - chunk.object?.replace(/\.chunk$/, '') ?? context.responseObject; - context.responseCreated = chunk.created ?? context.responseCreated; - context.usage = chunk.usage ?? context.usage; - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - context.role = delta?.role ?? context.role; - content += delta?.content ?? ''; - reasoning += delta?.reasoning_content ?? delta?.reasoning ?? ''; - - // A tool-call frame is held rather than forwarded: if the turn turns out - // to invoke a server tool, the call has to be answered locally instead - // of being handed to the client as an unresolved call. Anything else the - // delta carried — most importantly the text the model wrote before - // deciding to search — still belongs to the visible turn, so it is - // re-emitted without the tool call. - if (delta?.tool_calls?.length) { - toolCallDeltas.push(...delta.tool_calls); - frames.push(frame); - - const visibleDelta = { ...delta }; - delete visibleDelta.tool_calls; - - if (Object.keys(visibleDelta).length) { - const visible = JSON.stringify({ - ...chunk, - choices: [{ ...choice, delta: visibleDelta, finish_reason: null }], - }); - emitRaw(`data: ${visible}`); - } - return; - } - - if (choice?.finish_reason === 'tool_calls') { - frames.push(frame); - return; - } - } catch { - emitRaw(frame); - return; - } - - emitRaw(frame); - }; - - while (true) { - const chunk = await reader.read(); - if (!canContinue()) break; - if (chunk.done) break; - buffer += decoder.decode(chunk.value, { stream: true }); - const split = buffer.split(/\r?\n\r?\n/); - buffer = split.pop() ?? ''; - split.forEach(inspectFrame); - } - - if (buffer.trim()) inspectFrame(buffer); - reader.releaseLock(); - onReader?.(null); - - const toolCalls = aggregateStreamingToolCalls(toolCallDeltas); - const isLocalCall = (toolCall: ChatCompletionToolCall): boolean => - isLocalServerToolCall({ - fetchProvider, - ownedNames, - searchProvider, - toolCall, - }); - - return { - content, - frames, - localCalls: toolCalls.filter(isLocalCall), - reasoning, - remainingCalls: toolCalls.filter((toolCall) => !isLocalCall(toolCall)), - role: context.role, - toolCalls, - usage: context.usage, - }; -}; - const createInlineServerToolStream = async ({ body, callbacks, @@ -1803,55 +723,15 @@ export const executeWebSearchLoop = async ({ typeof message?.content === 'string' ? message.content.trim() : ''; const iterationReasoning = readReasoning(message).trim(); - const invocations = localCalls.map( - (toolCall, index): ServerToolInvocation => - isWebFetchToolCall(toolCall) - ? { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: extractFetchQuery(toolCall.function?.arguments), - type: 'web_fetch', - } - : { - id: toolCall.id ?? `server_tool_${iteration}_${index}`, - input: { - query: extractSearchQuery(toolCall.function?.arguments), - }, - type: 'web_search', - }, - ); - invocations.forEach((invocation) => callbacks?.onCall?.(invocation)); - - const results = await Promise.all( - invocations.map(async (invocation) => { - if (invocation.type === 'web_fetch') { - const result = await runWebFetchResult({ - provider: fetchProvider, - query: invocation.input, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - } - - const result = await runWebSearchResult({ - provider: searchProvider, - query: invocation.input.query, - }); - const execution: ServerToolExecution = { ...invocation, result }; - callbacks?.onResult?.(execution); - - return { - content: result.content, - execution, - tool_call_id: invocation.id, - }; - }), + const invocations = localCalls.map((toolCall, index) => + buildServerToolInvocation(toolCall, iteration, index), ); + const results = await executeServerToolInvocations({ + callbacks, + fetchProvider, + invocations, + searchProvider, + }); executions.push(...results.map((result) => result.execution)); // A turn mixing server tools with client-side calls cannot be continued @@ -2018,126 +898,27 @@ export const executeWebSearchLoop = async ({ }; }; -/** - * Replays a buffered completion as chat-completion SSE. Used only after a - * streaming request actually invokes a server-executed tool; ordinary answers - * keep the upstream response untouched. - */ -export const synthesizeChatCompletionStream = ( - payload: ChatCompletionPayload, - fallbackModel: string, -): Response => { - const encoder = new TextEncoder(); - const choice = payload.choices?.[0]; - const message = choice?.message; - const created = - typeof payload.created === 'number' - ? payload.created - : Math.floor(Date.now() / 1000); - const model = - typeof payload.model === 'string' && payload.model - ? payload.model - : fallbackModel; - const id = - typeof payload.id === 'string' && payload.id - ? payload.id - : `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`; - - const enqueue = ( - controller: ReadableStreamDefaultController, - chunk: JsonRecord, - ): void => { - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ - ...chunk, - created, - id, - model, - object: 'chat.completion.chunk', - })}\n\n`, - ), - ); - }; - - const stream = new ReadableStream({ - start: (controller) => { - enqueue(controller, { - choices: [{ delta: { role: 'assistant' }, index: 0 }], - }); - - const reasoning = message?.reasoning_content ?? message?.reasoning; - - if (reasoning) { - enqueue(controller, { - choices: [{ delta: { reasoning_content: reasoning }, index: 0 }], - }); - } - - const content = - typeof message?.content === 'string' ? message.content : ''; - - for ( - let offset = 0; - offset < content.length; - offset += STREAM_TEXT_CHUNK_LENGTH - ) { - enqueue(controller, { - choices: [ - { - delta: { - content: content.slice( - offset, - offset + STREAM_TEXT_CHUNK_LENGTH, - ), - }, - index: 0, - }, - ], - }); - } - - const passthroughCalls = (message?.tool_calls ?? []).map( - (toolCall, index) => ({ - ...toolCall, - id: toolCall.id ?? `call_${index}`, - index, - type: toolCall.type ?? 'function', - }), - ); - - if (passthroughCalls.length) { - enqueue(controller, { - choices: [{ delta: { tool_calls: passthroughCalls }, index: 0 }], - }); - } - - enqueue(controller, { - choices: [ - { - delta: {}, - finish_reason: - choice?.finish_reason ?? - (passthroughCalls.length ? 'tool_calls' : 'stop'), - index: 0, - }, - ], - }); - - if (payload.usage !== undefined && payload.usage !== null) { - enqueue(controller, { - choices: [], - usage: payload.usage, - }); - } - - controller.enqueue(encodeDoneFrame()); - controller.close(); - }, - }); - - return createSseResponse(stream, { - headers: { 'Access-Control-Allow-Origin': '*' }, - status: 200, - }); -}; +export type { + ChatCompletionMessage, + ChatCompletionPayload, + ChatCompletionToolCall, + ServerToolCallbacks, + ServerToolExecution, + ServerToolInvocation, + ServerToolLoopResult, + ServerToolStreamEvent, + ServerToolTurn, + ServerToolUpstreamMode, +} from './server-tool-types'; + +export { + attachServerToolExecutions, + attachServerToolTurns, + getServerToolExecutions, + getServerToolStreamEvent, + getServerToolTurns, +} from './server-tool-execution'; + +export { synthesizeChatCompletionStream } from './server-tool-sse'; + +export { withIntermediateTurns } from './server-tool-turns'; From ce5b4da41cbaeaee09de26505b1f7d7831d788f6 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 22:00:22 +0800 Subject: [PATCH 3/7] refactor(anthropic): split the Messages mapper into focused modules anthropic.ts was 1734 lines mixing protocol types, content conversion, request and response translation, streaming and error shaping. Each of those now has its own module, leaving the route entry point and the re-exports that keep the public surface unchanged. --- lib/server/proxy/anthropic-content.ts | 145 ++ lib/server/proxy/anthropic-errors.ts | 49 + lib/server/proxy/anthropic-request.ts | 451 +++++++ lib/server/proxy/anthropic-response.ts | 255 ++++ lib/server/proxy/anthropic-stream.ts | 636 +++++++++ lib/server/proxy/anthropic-types.ts | 184 +++ lib/server/proxy/anthropic.ts | 1682 +----------------------- 7 files changed, 1740 insertions(+), 1662 deletions(-) create mode 100644 lib/server/proxy/anthropic-content.ts create mode 100644 lib/server/proxy/anthropic-errors.ts create mode 100644 lib/server/proxy/anthropic-request.ts create mode 100644 lib/server/proxy/anthropic-response.ts create mode 100644 lib/server/proxy/anthropic-stream.ts create mode 100644 lib/server/proxy/anthropic-types.ts diff --git a/lib/server/proxy/anthropic-content.ts b/lib/server/proxy/anthropic-content.ts new file mode 100644 index 0000000..aafcc48 --- /dev/null +++ b/lib/server/proxy/anthropic-content.ts @@ -0,0 +1,145 @@ +import { stringifyContent } from '../shared/content'; +import type { + AnthropicContentBlock, + AnthropicImageSource, + ChatContent, + ChatContentPart, + ChatImageBlock, + ChatTextBlock, + ChatTextContent, +} from './anthropic-types'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +export const createAnthropicId = (prefix: string): string => { + return `${prefix}_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const mapTextPartsToChatContent = ( + parts: Array, +): ChatTextContent => { + const textParts = parts.filter((part) => + typeof part === 'string' ? part.length > 0 : part.text.length > 0, + ); + const hasStructuredText = textParts.some((part) => typeof part !== 'string'); + + if (!hasStructuredText) { + return textParts.join('\n'); + } + + return textParts.flatMap((part, index) => [ + ...(index > 0 ? [{ type: 'text' as const, text: '\n' }] : []), + typeof part === 'string' ? { type: 'text' as const, text: part } : part, + ]); +}; + +/** + * Builds the `image_url` value for an Anthropic image block. Base64 sources + * become a data URI because the upstream Chat/Responses APIs expect a URL; + * `url` sources pass through untouched. Returns undefined for an unusable + * source so the caller can fall back to a text placeholder rather than + * emitting a block the upstream would reject. + */ +export const buildChatImageUrl = ( + source: AnthropicImageSource | undefined, +): string | undefined => { + if (!source || typeof source !== 'object') { + return undefined; + } + + if (source.type === 'url' || (!source.data && source.url)) { + return typeof source.url === 'string' && source.url + ? source.url + : undefined; + } + + if (typeof source.data !== 'string' || !source.data) { + return undefined; + } + + const mediaType = + typeof source.media_type === 'string' && source.media_type + ? source.media_type + : 'image/png'; + + return `data:${mediaType};base64,${source.data}`; +}; + +/** + * Like `mapTextPartsToChatContent`, but keeps image parts as real image + * blocks instead of collapsing them into text. Falls back to the text-only + * result when nothing resolved to an image. + */ +export const mapContentPartsToChat = ( + parts: ChatContentPart[], +): ChatContent => { + const hasImage = parts.some( + (part) => typeof part === 'object' && part.type === 'image_url', + ); + + if (!hasImage) { + return mapTextPartsToChatContent( + parts.filter( + (part): part is string | ChatTextBlock => + typeof part === 'string' || part.type === 'text', + ), + ); + } + + const blocks: Array = []; + let pendingText: Array = []; + + const flushText = (): void => { + if (!pendingText.length) { + return; + } + const textContent = mapTextPartsToChatContent(pendingText); + if (typeof textContent === 'string') { + blocks.push({ type: 'text', text: textContent }); + } else { + blocks.push(...textContent); + } + pendingText = []; + }; + + for (const part of parts) { + if (typeof part === 'object' && part.type === 'image_url') { + flushText(); + blocks.push(part); + continue; + } + pendingText.push(part); + } + + flushText(); + + return blocks; +}; + +export const extractSystemText = ( + system: string | AnthropicContentBlock[] | undefined, +): ChatTextContent => { + if (!system) { + return ''; + } + + if (typeof system === 'string') { + return system; + } + + return mapTextPartsToChatContent( + system.map((block) => { + if (block.type === 'text') { + const text = block.text ?? ''; + + return block.cache_control + ? { type: 'text', text, cache_control: block.cache_control } + : text; + } + + return stringifyContent(block); + }), + ); +}; diff --git a/lib/server/proxy/anthropic-errors.ts b/lib/server/proxy/anthropic-errors.ts new file mode 100644 index 0000000..fce19df --- /dev/null +++ b/lib/server/proxy/anthropic-errors.ts @@ -0,0 +1,49 @@ +import { extractErrorMessage } from '../shared/http'; + +export const anthropicErrorType = (status: number): string => + status === 401 + ? 'authentication_error' + : status === 403 + ? 'permission_error' + : status === 404 + ? 'not_found_error' + : status === 413 + ? 'request_too_large' + : status === 429 + ? 'rate_limit_error' + : status === 529 + ? 'overloaded_error' + : status >= 500 + ? 'api_error' + : 'invalid_request_error'; + +export const getUpstreamErrorMessage = async ( + response: Response, +): Promise => { + const text = await response.text(); + if (!text) return 'Upstream CodeBuddy request failed'; + + try { + return extractErrorMessage(JSON.parse(text) as unknown) ?? text; + } catch { + return text; + } +}; + +export const createAnthropicError = ( + status: number, + message: string, +): Response => { + const type = anthropicErrorType(status); + + return Response.json( + { + type: 'error', + error: { + type, + message, + }, + }, + { status }, + ); +}; diff --git a/lib/server/proxy/anthropic-request.ts b/lib/server/proxy/anthropic-request.ts new file mode 100644 index 0000000..bce63f8 --- /dev/null +++ b/lib/server/proxy/anthropic-request.ts @@ -0,0 +1,451 @@ +import { + getDefaultModel, + isWebFetchEnabled, + isWebSearchEnabled, +} from '../domain/config'; +import { stringifyContent } from '../shared/content'; +import { + markServerTool, + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_FETCH_TOOL_TYPE_PREFIX, + WEB_SEARCH_TOOL_NAME, + WEB_SEARCH_TOOL_TYPE_PREFIX, +} from '../search/tool'; +import { + buildChatImageUrl, + createAnthropicId, + extractSystemText, + mapContentPartsToChat, +} from './anthropic-content'; +import type { + AnthropicContentBlock, + AnthropicMessage, + AnthropicMessagesRequestBody, + AnthropicTool, + ChatContentPart, + ChatImageBlock, + ChatMessage, +} from './anthropic-types'; + +// --------------------------------------------------------------------------- +// Request translation: Anthropic → OpenAI +// --------------------------------------------------------------------------- + +export const decodeOpaqueServerToolContent = (value: unknown): unknown => { + if (typeof value !== 'string' || !value) { + return null; + } + + try { + const binary = atob(value); + const bytes = Uint8Array.from(binary, (character) => + character.charCodeAt(0), + ); + + return JSON.parse(new TextDecoder().decode(bytes)) as unknown; + } catch { + return null; + } +}; + +export const formatAnthropicServerToolResult = ( + block: AnthropicContentBlock, +): string => { + if (block.type === 'web_search_tool_result' && Array.isArray(block.content)) { + return block.content + .map((value, index) => { + const item = + value && typeof value === 'object' + ? (value as Record) + : {}; + const decoded = decodeOpaqueServerToolContent(item.encrypted_content); + const source = + decoded && typeof decoded === 'object' + ? (decoded as Record) + : item; + const title = String(source.title ?? item.title ?? '').trim(); + const url = String(source.url ?? item.url ?? '').trim(); + const text = String( + source.content ?? source.snippet ?? source.text ?? '', + ).trim(); + + return [ + `${index + 1}. ${title || url || 'Search result'}`, + ...(url ? [`URL: ${url}`] : []), + ...(text ? [text] : []), + ].join('\n'); + }) + .join('\n\n'); + } + + if ( + block.type === 'web_fetch_tool_result' && + block.content && + typeof block.content === 'object' + ) { + const result = block.content as Record; + const document = + result.content && typeof result.content === 'object' + ? (result.content as Record) + : null; + const source = + document?.source && typeof document.source === 'object' + ? (document.source as Record) + : null; + const url = typeof result.url === 'string' ? result.url : ''; + const text = typeof source?.data === 'string' ? source.data : ''; + + return [url, text].filter(Boolean).join('\n\n'); + } + + // Nested images are emitted as real image parts by + // `collectAnthropicNestedImages`, so they are excluded here to keep their + // base64 payload out of the text. + if (Array.isArray(block.content)) { + return stringifyContent( + block.content.filter((value) => { + return !( + value && + typeof value === 'object' && + (value as AnthropicContentBlock).type === 'image' + ); + }), + ); + } + + return typeof block.content === 'string' + ? block.content + : stringifyContent(block.content); +}; + +/** + * Images nested inside a `tool_result` content array, e.g. a screenshot a tool + * returned. The outer block is handled by the `tool_result` branch, whose + * formatter stringifies nested content — so without extracting them here the + * model would receive the base64 payload as text. + */ +export const collectAnthropicNestedImages = ( + block: AnthropicContentBlock, +): ChatImageBlock[] => { + if (!Array.isArray(block.content)) { + return []; + } + + return block.content.flatMap((value): ChatImageBlock[] => { + if (!value || typeof value !== 'object') { + return []; + } + + const nested = value as AnthropicContentBlock; + + if (nested.type !== 'image') { + return []; + } + + const imageUrl = buildChatImageUrl(nested.source); + + return imageUrl + ? [{ type: 'image_url', image_url: { url: imageUrl } }] + : []; + }); +}; + +export const mapAnthropicContentToChat = ( + content: string | AnthropicContentBlock[], + role: 'user' | 'assistant', +): ChatMessage[] => { + if (typeof content === 'string') { + return [{ role, content }]; + } + + const parts: ChatContentPart[] = []; + const toolCalls: Array<{ + id: string; + type: string; + function: { + name: string; + arguments: string; + }; + }> = []; + const toolResults: ChatMessage[] = []; + const messages: ChatMessage[] = []; + /** + * Reasoning recovered from thinking blocks in this assistant message. + * + * Attached to the message the blocks belong to rather than sent on its own: + * a bare reasoning entry is not a valid chat message, and the upstream needs + * the reasoning alongside the text and tool calls it produced. + */ + let pendingReasoning = ''; + const flushAssistantMessage = (): void => { + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; + + if (!toolCalls.length && !hasContent && !pendingReasoning) { + return; + } + + messages.push({ + role: 'assistant', + content: hasContent ? content : null, + ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}), + // `reasoning` is the field the CodeBuddy chat upstream round-trips. It + // is not part of the OpenAI schema, but the upstream accepts it and + // ignoring an unknown field costs nothing if it ever stops doing so. + ...(pendingReasoning ? { reasoning: pendingReasoning } : {}), + }); + parts.length = 0; + toolCalls.length = 0; + pendingReasoning = ''; + }; + + for (const block of content) { + if (block.type === 'text') { + const text = block.text ?? ''; + + parts.push( + block.cache_control + ? { type: 'text', text, cache_control: block.cache_control } + : text, + ); + } else if (block.type === 'tool_use' || block.type === 'server_tool_use') { + toolCalls.push({ + id: block.id ?? createAnthropicId('toolu'), + type: 'function', + function: { + name: block.name ?? 'unknown', + arguments: JSON.stringify(block.input ?? {}), + }, + }); + } else if ( + block.type === 'tool_result' || + block.type === 'web_search_tool_result' || + block.type === 'web_fetch_tool_result' + ) { + const nestedImages = collectAnthropicNestedImages(block); + + const resultMessage: ChatMessage = { + role: 'tool', + content: nestedImages.length + ? mapContentPartsToChat([ + formatAnthropicServerToolResult(block), + ...nestedImages, + ]) + : formatAnthropicServerToolResult(block), + tool_call_id: block.tool_use_id ?? '', + }; + + if (role === 'assistant') { + flushAssistantMessage(); + messages.push(resultMessage); + } else { + toolResults.push(resultMessage); + } + } else if ( + block.type === 'thinking' || + block.type === 'redacted_thinking' + ) { + // Replaying prior-turn reasoning is required inside a tool-use turn and + // harmless elsewhere, so recover it instead of dropping it. + // + // The `thinking` field carries the reasoning. A `signature` is only ever + // read when it is one we minted on the Responses path; a genuine + // Anthropic signature is ciphertext, and forwarding it upstream would put + // gibberish where reasoning belongs. + // + // `redacted_thinking` has no readable text at all, only `data`, but must + // still be matched here: without this branch it fell through to + // `stringifyContent` and the model received a JSON dump of the opaque + // payload as if it were user prose. + const reasoning = block.thinking ?? ''; + + if (reasoning) { + pendingReasoning = pendingReasoning + ? `${pendingReasoning}${reasoning}` + : reasoning; + } + } else if (block.type === 'image' && block.source) { + // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url', + // media_type, data | url } }`. Emit a real image block so the upstream + // model sees the image; without this branch the block fell through to + // `stringifyContent` and the model received a JSON dump of the base64 + // payload as text. An `image` block with no `source` is not a real + // Anthropic image, so it keeps the generic stringified handling. + const imageUrl = buildChatImageUrl(block.source); + + parts.push( + imageUrl + ? { + type: 'image_url', + image_url: { url: imageUrl }, + // Preserve an explicit cache breakpoint, matching how text + // blocks carry `cache_control` through. Without this the + // requested breakpoint is dropped and `applyPromptCacheControl` + // falls back to its own automatic placement. + ...(block.cache_control + ? { cache_control: block.cache_control } + : {}), + } + : stringifyContent(block), + ); + } else { + parts.push(stringifyContent(block)); + } + } + + if (role === 'user') { + messages.push(...toolResults); + const content = mapContentPartsToChat(parts); + const hasContent = typeof content === 'string' ? content.length > 0 : true; + if (hasContent) { + messages.push({ role: 'user', content }); + } + } else { + flushAssistantMessage(); + } + + return messages; +}; + +export const mapAnthropicMessagesToChat = ( + messages: AnthropicMessage[], +): ChatMessage[] => { + const result: ChatMessage[] = []; + + for (const msg of messages) { + const mapped = mapAnthropicContentToChat(msg.content, msg.role); + + if (mapped.length === 0) { + continue; + } + + for (const item of mapped) { + result.push({ + ...item, + }); + } + } + + return result; +}; + +export const mapAnthropicToolsToChat = ( + tools: AnthropicTool[] | undefined, +): unknown[] | undefined => { + if (!tools?.length) { + return undefined; + } + + return tools.map((tool) => { + const mapped = { + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: tool.input_schema, + }, + }; + const normalizedType = normalizeToolName(tool.type ?? ''); + const serverDeclared = [ + WEB_SEARCH_TOOL_TYPE_PREFIX, + WEB_FETCH_TOOL_TYPE_PREFIX, + ].some((prefix) => normalizedType.startsWith(normalizeToolName(prefix))); + + return serverDeclared ? markServerTool(mapped) : mapped; + }); +}; + +export const shouldBridgeAnthropicServerTools = async ( + tools: AnthropicTool[] | undefined, +): Promise => { + const names = new Set( + (tools ?? []).map((tool) => normalizeToolName(tool.name)), + ); + const [searchEnabled, fetchEnabled] = await Promise.all([ + names.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) + ? isWebSearchEnabled() + : false, + names.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) + ? isWebFetchEnabled() + : false, + ]); + + return searchEnabled || fetchEnabled; +}; + +export const mapAnthropicToolChoiceToChat = (toolChoice: unknown): unknown => { + if (!toolChoice || typeof toolChoice !== 'object') { + return toolChoice; + } + + const tc = toolChoice as { type?: string; name?: string }; + + if (tc.type === 'auto') { + return 'auto'; + } + + if (tc.type === 'any') { + return 'required'; + } + + if (tc.type === 'tool' && tc.name) { + return { + type: 'function', + function: { name: tc.name }, + }; + } + + if (tc.type === 'none') { + return 'none'; + } + + return toolChoice; +}; + +export const buildChatRequestBody = async ( + body: AnthropicMessagesRequestBody, +): Promise> => { + const systemText = extractSystemText(body.system); + const chatMessages = mapAnthropicMessagesToChat(body.messages ?? []); + + const messages: ChatMessage[] = []; + const disableParallelToolUse = + body.tool_choice && typeof body.tool_choice === 'object' + ? (body.tool_choice as { disable_parallel_tool_use?: unknown }) + .disable_parallel_tool_use + : undefined; + + if (systemText) { + messages.push({ role: 'system', content: systemText }); + } + + messages.push(...chatMessages); + + const result: Record = { + model: + typeof body.model === 'string' && body.model.trim() + ? body.model + : await getDefaultModel('claude-sonnet-4.6'), + messages, + stream: body.stream ?? false, + max_tokens: body.max_tokens, + temperature: body.temperature, + top_p: body.top_p, + stop: body.stop_sequences, + tools: mapAnthropicToolsToChat(body.tools), + tool_choice: mapAnthropicToolChoiceToChat(body.tool_choice), + parallel_tool_calls: + typeof disableParallelToolUse === 'boolean' + ? !disableParallelToolUse + : undefined, + }; + + // Pass through thinking/reasoning config so upstream models that support + // extended thinking can honor it. + if (body.thinking) { + result.thinking = body.thinking; + } + + return result; +}; diff --git a/lib/server/proxy/anthropic-response.ts b/lib/server/proxy/anthropic-response.ts new file mode 100644 index 0000000..dd0cca1 --- /dev/null +++ b/lib/server/proxy/anthropic-response.ts @@ -0,0 +1,255 @@ +import { createAnthropicId } from './anthropic-content'; +import type { + AnthropicContentBlock, + OpenAIChatResponse, + OpenAIUsage, +} from './anthropic-types'; +import type { ServerToolExecution, ServerToolTurn } from './web-search-loop'; + +// --------------------------------------------------------------------------- +// Response translation: OpenAI → Anthropic (non-streaming) +// --------------------------------------------------------------------------- + +export const mapOpenAIUsageToAnthropic = ( + usage: OpenAIUsage | undefined, + serverToolExecutions: ServerToolExecution[] = [], +): Record => { + const cacheCreationTokens = + usage?.prompt_tokens_details?.cache_creation_tokens ?? 0; + const cacheReadTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; + // prompt_tokens is the total prompt count including cached tokens. + // Anthropic reports cached/created tokens separately, so input_tokens + // must be the non-cache remainder to avoid double-counting. + const inputTokens = Math.max( + 0, + (usage?.prompt_tokens ?? 0) - cacheCreationTokens - cacheReadTokens, + ); + const outputTokens = usage?.completion_tokens ?? 0; + + const mapped: Record> = { + input_tokens: inputTokens, + output_tokens: outputTokens, + cache_creation_input_tokens: cacheCreationTokens, + cache_read_input_tokens: cacheReadTokens, + }; + + if (serverToolExecutions.length) { + mapped.server_tool_use = { + web_search_requests: serverToolExecutions.filter( + (execution) => execution.type === 'web_search', + ).length, + web_fetch_requests: serverToolExecutions.filter( + (execution) => execution.type === 'web_fetch', + ).length, + }; + } + + return mapped; +}; + +export const encodeOpaqueServerToolContent = (value: unknown): string => { + const bytes = new TextEncoder().encode(JSON.stringify(value)); + let binary = ''; + + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + + return btoa(binary); +}; + +export const buildAnthropicServerToolBlocks = ( + execution: ServerToolExecution, +): AnthropicContentBlock[] => { + const id = createAnthropicId('srvtoolu'); + const result = + execution.type === 'web_search' + ? { + type: 'web_search_tool_result', + tool_use_id: id, + content: execution.result.results.map((item) => ({ + type: 'web_search_result', + url: item.url ?? '', + title: item.title ?? '', + encrypted_content: encodeOpaqueServerToolContent(item), + })), + } + : { + type: 'web_fetch_tool_result', + tool_use_id: id, + content: { + type: 'web_fetch_result', + url: execution.result.url ?? execution.input.url, + content: { + type: 'document', + source: { + type: 'text', + media_type: 'text/plain', + data: execution.result.content, + }, + }, + }, + }; + + return [ + { + type: 'server_tool_use', + id, + name: execution.type, + input: execution.input, + }, + result, + ]; +}; + +export const buildAllAnthropicServerToolBlocks = ( + executions: ServerToolExecution[], +): AnthropicContentBlock[] => + executions.flatMap(buildAnthropicServerToolBlocks); + +/** + * We do not mint a `signature` on this path. It would have to duplicate the + * `thinking` text to be replayable, which puts the reasoning on the wire twice + * for callers that count it — and the block already replays fine: Anthropic + * clients echo `thinking` back, which is what inbound handling reads. + */ +export const buildThinkingBlock = ( + thinking: string, +): AnthropicContentBlock => ({ + type: 'thinking', + thinking, +}); + +/** + * Lays a server-tool turn out the way Anthropic does: each hop contributes its + * own thinking and text, followed by the tool blocks that hop triggered. + * + * `turns` carries the per-hop grouping the OpenAI-shaped payload cannot. Under + * that protocol a multi-hop turn collapses into one `content` string and one + * `reasoning_content` string, which loses where one hop's reasoning ends and the + * next begins — so the grouping has to be recovered before it is joined, which + * is why the loop emits it alongside the strings rather than this file + * reconstructing it. + * + * Anthropic's own server tools run multiple hops inside one assistant message, + * and a client replaying that message expects `[thinking] [text] [tool_use] + * [tool_result] [thinking] [text]`. Gathering the blocks by kind instead — every + * tool ahead of all the prose — puts each search before the reasoning that asked + * for it and merges hops that were never contiguous. + */ +export const buildAnthropicTurnBlocks = ( + turns: ServerToolTurn[], +): AnthropicContentBlock[] => { + const blocks: AnthropicContentBlock[] = []; + + turns.forEach((turn) => { + if (turn.reasoning) { + blocks.push(buildThinkingBlock(turn.reasoning)); + } + + if (turn.text) { + blocks.push({ type: 'text', text: turn.text }); + } + + blocks.push(...buildAllAnthropicServerToolBlocks(turn.executions)); + }); + + return blocks; +}; + +export const mapOpenAIResponseToAnthropic = ( + openaiResponse: OpenAIChatResponse, + model: string, + serverToolExecutions: ServerToolExecution[] = [], + turns?: ServerToolTurn[], +): Record => { + const choice = openaiResponse.choices?.[0]; + const message = choice?.message; + + // Thinking / reasoning content + const reasoningText = message?.reasoning_content ?? message?.reasoning ?? ''; + + // Text content + const textContent = + typeof message?.content === 'string' ? message.content : ''; + + // With per-hop grouping the turns already hold every block in order, prose + // included. Without it — no server tool ran, or a path that never grouped the + // hops — fall back to Anthropic's own order: thinking and text first, then + // the server-tool blocks they led to. + const contentBlocks: AnthropicContentBlock[] = turns + ? buildAnthropicTurnBlocks(turns) + : []; + + if (!turns) { + if (reasoningText) { + contentBlocks.push(buildThinkingBlock(reasoningText)); + } + + if (textContent) { + contentBlocks.push({ type: 'text', text: textContent }); + } + + contentBlocks.push( + ...buildAllAnthropicServerToolBlocks(serverToolExecutions), + ); + } + + // Tool calls + const toolCalls = message?.tool_calls ?? []; + + for (const call of toolCalls) { + let input: unknown = {}; + + try { + input = JSON.parse(call.function?.arguments ?? '{}'); + } catch { + input = {}; + } + + contentBlocks.push({ + type: 'tool_use', + id: call.id ?? createAnthropicId('toolu'), + name: call.function?.name ?? 'unknown', + input, + }); + } + + const stopReason = mapFinishReasonToAnthropic( + choice?.finish_reason, + toolCalls.length > 0, + ); + + return { + id: openaiResponse.id ?? createAnthropicId('msg'), + type: 'message', + role: 'assistant', + model, + content: contentBlocks, + stop_reason: stopReason, + stop_sequence: null, + usage: mapOpenAIUsageToAnthropic( + openaiResponse.usage, + serverToolExecutions, + ), + }; +}; + +export const mapFinishReasonToAnthropic = ( + finishReason: string | null | undefined, + hasToolCalls: boolean, +): string => { + if (hasToolCalls || finishReason === 'tool_calls') { + return 'tool_use'; + } + + if (finishReason === 'length') { + return 'max_tokens'; + } + + if (finishReason === 'stop' || !finishReason) { + return 'end_turn'; + } + + return 'end_turn'; +}; diff --git a/lib/server/proxy/anthropic-stream.ts b/lib/server/proxy/anthropic-stream.ts new file mode 100644 index 0000000..3f7363b --- /dev/null +++ b/lib/server/proxy/anthropic-stream.ts @@ -0,0 +1,636 @@ +import type { NextRequest } from 'next/server'; + +import type { DebugTrace } from '../domain/debug'; +import { createSseResponse } from '../shared/sse'; +import { + anthropicStreamErrorChunks, + createStreamCloser, + toUpstreamTimeoutMessage, +} from '../shared/upstream-timeout'; +import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; +import { createAnthropicId } from './anthropic-content'; +import { + anthropicErrorType, + getUpstreamErrorMessage, +} from './anthropic-errors'; +import { + buildAnthropicServerToolBlocks, + mapFinishReasonToAnthropic, + mapOpenAIUsageToAnthropic, +} from './anthropic-response'; +import { MAX_STREAM_FRAME_LENGTH } from './anthropic-types'; +import type { + OpenAIStreamChunk, + OpenAIStreamError, + OpenAIUsage, + StreamingToolUseState, +} from './anthropic-types'; +import { + getServerToolExecutions, + getServerToolStreamEvent, + type ServerToolExecution, +} from './web-search-loop'; + +// --------------------------------------------------------------------------- +// Response translation: OpenAI SSE → Anthropic SSE (streaming) +// --------------------------------------------------------------------------- + +export const mapOpenAIStreamToAnthropicSSE = ( + upstreamResponse: Response, + model: string, + options?: { + emitMessageStart?: boolean; + initialContentBlockCount?: number; + messageId?: string; + serverToolExecutions?: ServerToolExecution[]; + }, +): Response => { + if (!upstreamResponse.body) { + return createSseResponse(null, { status: upstreamResponse.status }); + } + + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + + const messageId = options?.messageId ?? createAnthropicId('msg'); + const serverToolExecutions = + options?.serverToolExecutions ?? getServerToolExecutions(upstreamResponse); + const serverToolUseIds = new Map(); + const toolUseStates = new Map(); + let nextToolIndex = 0; + let started = options?.emitMessageStart === false; + let thinkingStarted = false; + let thinkingBlockIndex = -1; + let textStarted = false; + let textBlockIndex = -1; + // Tracks how many content blocks (thinking + text) have been opened + // so tool_use blocks get correct sequential indices even after the + // prior blocks are closed mid-stream. + let contentBlockCount = options?.initialContentBlockCount ?? 0; + let finishReason: string | null = null; + let hasToolCalls = false; + let usage: OpenAIUsage | undefined; + + const enqueueEvent = (event: Record): void => { + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ); + }; + + let controller: ReadableStreamDefaultController; + + // Close any open text/thinking block before starting a tool_use block. + // Anthropic streaming requires each block to be stopped before the next. + const closeOpenTextBlocks = (): void => { + if (thinkingStarted) { + // Anthropic emits the signature last, just before the block closes — but + // we do not send one here: the reasoning already went out as + // `thinking_delta`s, and duplicating it into a signature would put the + // text on the wire twice for callers that count it. See + // `buildThinkingBlock`. + enqueueEvent({ + type: 'content_block_stop', + index: thinkingBlockIndex, + }); + thinkingStarted = false; + } + + if (textStarted) { + enqueueEvent({ + type: 'content_block_stop', + index: textBlockIndex, + }); + textStarted = false; + } + }; + + const processChunk = (chunk: OpenAIStreamChunk): void => { + if (!started) { + started = true; + enqueueEvent({ + type: 'message_start', + message: { + id: messageId, + type: 'message', + role: 'assistant', + content: [], + model, + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: chunk.usage?.prompt_tokens ?? 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + }); + } + + if (chunk.usage) { + usage = chunk.usage; + } + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (!delta) { + return; + } + + // Reasoning / thinking content + const reasoningText = delta.reasoning_content ?? delta.reasoning ?? ''; + + if (reasoningText) { + if (!thinkingStarted) { + thinkingBlockIndex = contentBlockCount; + thinkingStarted = true; + enqueueEvent({ + type: 'content_block_start', + index: thinkingBlockIndex, + content_block: { + type: 'thinking', + thinking: '', + }, + }); + contentBlockCount++; + } + + enqueueEvent({ + type: 'content_block_delta', + index: thinkingBlockIndex, + delta: { + type: 'thinking_delta', + thinking: reasoningText, + }, + }); + } + + // Text content + if (delta.content) { + if (!textStarted) { + // Close the thinking block before starting text so Anthropic + // stream consumers see properly ordered, non-overlapping blocks. + closeOpenTextBlocks(); + + textBlockIndex = contentBlockCount; + textStarted = true; + enqueueEvent({ + type: 'content_block_start', + index: textBlockIndex, + content_block: { + type: 'text', + text: '', + }, + }); + contentBlockCount++; + } + + enqueueEvent({ + type: 'content_block_delta', + index: textBlockIndex, + delta: { + type: 'text_delta', + text: delta.content, + }, + }); + } + + // Tool calls + if (delta.tool_calls?.length) { + hasToolCalls = true; + + // Anthropic streaming requires each content block to be closed + // before the next one starts. If we already opened a text or + // thinking block, close it now so the tool_use block is well-formed. + closeOpenTextBlocks(); + + for (const call of delta.tool_calls) { + const callId = call.id ?? `toolu_${nextToolIndex}`; + const key = callId; + + if (!toolUseStates.has(key)) { + const blockIndex = contentBlockCount + nextToolIndex; + + toolUseStates.set(key, { + id: callId, + name: '', + input: '', + index: blockIndex, + started: false, + blockEmitted: false, + }); + nextToolIndex++; + } + + const state = toolUseStates.get(key)!; + + // Accumulate name fragments (upstream may stream the function + // name across multiple deltas, e.g. "look" + "up"). + if (call.function?.name) { + state.name += call.function.name; + } + + // Emit content_block_start lazily — once we have a name and at + // least one arguments fragment, so the block header carries the + // full tool name instead of a partial fragment. + if (!state.blockEmitted && state.name && call.function?.arguments) { + state.blockEmitted = true; + enqueueEvent({ + type: 'content_block_start', + index: state.index, + content_block: { + type: 'tool_use', + id: state.id, + name: state.name, + input: {}, + }, + }); + } + + if (call.function?.arguments) { + state.input += call.function.arguments; + enqueueEvent({ + type: 'content_block_delta', + index: state.index, + delta: { + type: 'input_json_delta', + partial_json: call.function.arguments, + }, + }); + } + } + } + + if (choice?.finish_reason) { + finishReason = choice.finish_reason; + } + }; + + const finalize = (): void => { + // Close any remaining open text/thinking blocks. + closeOpenTextBlocks(); + + // Close tool use blocks + for (const [, state] of toolUseStates) { + // If the block start was never emitted (e.g. name-only deltas + // with no arguments), emit it now so the block is well-formed. + if (!state.blockEmitted) { + state.blockEmitted = true; + enqueueEvent({ + type: 'content_block_start', + index: state.index, + content_block: { + type: 'tool_use', + id: state.id, + name: state.name || 'unknown', + input: {}, + }, + }); + } + + enqueueEvent({ + type: 'content_block_stop', + index: state.index, + }); + } + + const stopReason = mapFinishReasonToAnthropic(finishReason, hasToolCalls); + + enqueueEvent({ + type: 'message_delta', + delta: { + stop_reason: stopReason, + stop_sequence: null, + }, + usage: mapOpenAIUsageToAnthropic(usage, serverToolExecutions), + }); + + enqueueEvent({ + type: 'message_stop', + }); + }; + + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + let streamRejected = false; + const closer = createStreamCloser(); + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + const stream = new ReadableStream({ + start: (ctrl) => { + controller = ctrl; + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + const rejectStream = ( + message = 'Upstream SSE frame exceeds the maximum size', + status?: number, + ): void => { + streamRejected = true; + enqueueEvent({ + type: 'error', + error: { + // An upstream status names the failure precisely, so it decides + // the type: 429 has to arrive as `rate_limit_error` or a client + // that retries on that type alone stops retrying an exhausted + // quota. Without one, fall back to the message: an oversized frame + // is a malformed stream (`invalid_request_error`), while an + // upstream deadline is the server failing (`api_error`, the type + // clients treat as retryable). + type: + typeof status === 'number' + ? anthropicErrorType(status) + : message.includes('did not produce output') + ? 'api_error' + : 'invalid_request_error', + message, + }, + }); + }; + + const flushFrames = (frames: string[]): void => { + for (const frame of frames) { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + rejectStream(); + return; + } + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + continue; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + try { + const chunk = JSON.parse(raw) as OpenAIStreamChunk; + const serverToolEvent = getServerToolStreamEvent(chunk); + + if (serverToolEvent?.phase === 'call') { + closeOpenTextBlocks(); + const index = contentBlockCount++; + const toolUseId = createAnthropicId('srvtoolu'); + serverToolUseIds.set(serverToolEvent.invocation.id, toolUseId); + enqueueEvent({ + type: 'content_block_start', + index, + content_block: { + type: 'server_tool_use', + id: toolUseId, + name: serverToolEvent.invocation.type, + input: {}, + }, + }); + enqueueEvent({ + type: 'content_block_delta', + index, + delta: { + type: 'input_json_delta', + partial_json: JSON.stringify( + serverToolEvent.invocation.input, + ), + }, + }); + enqueueEvent({ type: 'content_block_stop', index }); + continue; + } + + if (serverToolEvent?.phase === 'result') { + closeOpenTextBlocks(); + serverToolExecutions.push(serverToolEvent.execution); + const index = contentBlockCount++; + const resultBlock = buildAnthropicServerToolBlocks( + serverToolEvent.execution, + )[1]; + enqueueEvent({ + type: 'content_block_start', + index, + content_block: { + ...resultBlock, + tool_use_id: serverToolUseIds.get( + serverToolEvent.execution.id, + ), + }, + }); + enqueueEvent({ type: 'content_block_stop', index }); + continue; + } + + const upstreamError = chunk as OpenAIStreamError; + if (upstreamError.error?.message) { + rejectStream( + upstreamError.error.message, + upstreamError.error.status, + ); + return; + } + processChunk(chunk); + } catch { + // Skip unparseable frames + } + } + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + if (buffer.trim()) { + flushFrames([buffer]); + } + if (!streamRejected) { + finalize(); + } + releaseReader(); + controller.close(); + return; + } + + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + rejectStream(); + } else { + flushFrames(frames); + } + if (streamRejected) { + try { + await reader!.cancel(); + } finally { + releaseReader(); + controller.close(); + } + return; + } + } + }; + + void pump().catch((error) => { + if (cancelled) return; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage === null) { + closer.mark(); + controller.error(error); + return; + } + + void reader?.cancel().then( + () => undefined, + () => undefined, + ); + releaseReader(); + closer.fail(controller, anthropicStreamErrorChunks(timeoutMessage)); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + releaseReader(); + } + }, + }); + + return createSseResponse(stream, { status: 200 }); +}; + +export const createAnthropicServerToolEventStream = ( + request: NextRequest, + chatBody: Record, + model: string, + debugTrace?: DebugTrace, +): Response => { + const encoder = new TextEncoder(); + const messageId = createAnthropicId('msg'); + let activeReader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + + const stream = new ReadableStream({ + start: (controller) => { + const enqueueEvent = (event: Record): void => { + if (cancelled) return; + controller.enqueue( + encoder.encode( + `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, + ), + ); + }; + + enqueueEvent({ + type: 'message_start', + message: { + id: messageId, + type: 'message', + role: 'assistant', + content: [], + model, + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }, + }, + }); + + const run = async (): Promise => { + const upstreamResponse = await proxyChatCompletions( + request, + chatBody as ChatRequestBody, + undefined, + debugTrace, + '/v1/messages', + // The findings reach the client as `web_search_tool_result` blocks, + // so the loop must not also fold them into the assistant text. + { emitStreamEvents: true, findingsAsStructuredBlocks: true }, + ); + + if (cancelled) { + await upstreamResponse.body?.cancel(); + return; + } + + if (!upstreamResponse.ok || !upstreamResponse.body) { + // A rate limit has to arrive as `rate_limit_error`, or a client that + // retries on that type alone will treat an exhausted quota as a + // generic failure and stop retrying — so the upstream status drives + // the event type even though the envelope is already streaming and + // the HTTP status cannot be changed. + const message = upstreamResponse.ok + ? 'Upstream request failed' + : await getUpstreamErrorMessage(upstreamResponse).catch( + () => 'Upstream request failed', + ); + + enqueueEvent({ + type: 'error', + error: { + type: anthropicErrorType(upstreamResponse.status), + message, + }, + }); + controller.close(); + return; + } + + const mappedResponse = mapOpenAIStreamToAnthropicSSE( + upstreamResponse, + model, + { + emitMessageStart: false, + initialContentBlockCount: 0, + messageId, + serverToolExecutions: [], + }, + ); + const reader = mappedResponse.body!.getReader(); + activeReader = reader; + + while (true) { + const { done, value } = await reader.read(); + if (cancelled) return; + if (done) break; + controller.enqueue(value); + } + + reader.releaseLock(); + activeReader = null; + controller.close(); + }; + + void run().catch((error) => { + if (!cancelled) controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + await activeReader?.cancel(reason); + activeReader?.releaseLock(); + activeReader = null; + }, + }); + + return createSseResponse(stream); +}; diff --git a/lib/server/proxy/anthropic-types.ts b/lib/server/proxy/anthropic-types.ts new file mode 100644 index 0000000..d1ebdee --- /dev/null +++ b/lib/server/proxy/anthropic-types.ts @@ -0,0 +1,184 @@ +// --------------------------------------------------------------------------- +// Anthropic Messages API types +// --------------------------------------------------------------------------- + +export const MAX_STREAM_FRAME_LENGTH = 1_000_000; + +export interface AnthropicImageSource { + type?: string; + media_type?: string; + data?: string; + url?: string; +} + +export interface AnthropicContentBlock { + type: string; + text?: string; + cache_control?: { type?: string }; + id?: string; + name?: string; + input?: unknown; + thinking?: string; + /** + * Accepted on inbound blocks but never sent by us — see + * `buildThinkingBlock`. Anthropic's signatures hold an encrypted copy of the + * reasoning; a client may replay one from a session it started elsewhere, and + * we skip those rather than forward ciphertext as if it were text. + */ + signature?: string; + /** Present on `redacted_thinking` blocks, which carry no readable text. */ + data?: string; + tool_use_id?: string; + content?: unknown; + source?: AnthropicImageSource; +} + +export interface AnthropicMessage { + role: 'user' | 'assistant'; + content: string | AnthropicContentBlock[]; +} + +export interface AnthropicTool { + name: string; + description?: string; + input_schema: Record; + type?: string; +} + +export interface AnthropicThinkingConfig { + type?: string; + budget_tokens?: number; +} + +export interface AnthropicMessagesRequestBody { + model?: string; + messages?: AnthropicMessage[]; + system?: string | AnthropicContentBlock[]; + max_tokens?: number; + temperature?: number; + top_p?: number; + top_k?: number; + stop_sequences?: string[]; + stream?: boolean; + tools?: AnthropicTool[]; + tool_choice?: unknown; + thinking?: AnthropicThinkingConfig; + metadata?: Record; +} + +// --------------------------------------------------------------------------- +// OpenAI response types (mirrors of codebuddy.ts internals) +// --------------------------------------------------------------------------- + +export interface OpenAIToolCall { + index?: number; + id?: string; + type?: string; + function?: { + arguments?: string; + name?: string; + }; +} + +export interface OpenAIChatMessage { + role?: string; + content?: unknown; + tool_calls?: OpenAIToolCall[]; + reasoning_content?: string; + reasoning?: string; +} + +export interface OpenAIChatChoice { + index?: number; + message?: OpenAIChatMessage; + delta?: OpenAIChatMessage; + finish_reason?: string | null; +} + +export interface OpenAIStreamError { + error?: { message?: string; status?: number }; +} + +export interface OpenAIUsage { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + prompt_tokens_details?: { + cached_tokens?: number; + cache_creation_tokens?: number; + }; + completion_tokens_details?: { + reasoning_tokens?: number; + }; +} + +export interface OpenAIChatResponse { + id?: string; + model?: string; + choices?: OpenAIChatChoice[]; + usage?: OpenAIUsage; +} + +export interface OpenAIStreamChunk { + id?: string; + model?: string; + choices?: OpenAIChatChoice[]; + usage?: OpenAIUsage; +} + +export interface ChatTextBlock { + cache_control?: { type?: string }; + text: string; + type: 'text'; +} + +/** + * An image part in the OpenAI Chat shape. Emitted in this shape rather than a + * native Anthropic one because the request is translated to Chat before it + * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the + * `responses` upstream converts it to `input_image`. + */ +export interface ChatImageBlock { + cache_control?: { type?: string }; + image_url: { url: string }; + type: 'image_url'; +} + +export type ChatContentPart = string | ChatTextBlock | ChatImageBlock; + +export type ChatContent = string | Array; + +/** + * Text-only content, used where images are not representable — the system + * prompt and the intermediate text-part buffer. + */ +export type ChatTextContent = string | ChatTextBlock[]; + +export interface ChatMessage { + role: string; + content: ChatContent | null; + tool_calls?: Array<{ + id: string; + type: string; + function: { + name: string; + arguments: string; + }; + }>; + tool_call_id?: string; + /** + * Prior-turn reasoning for this assistant message. Not part of the OpenAI + * schema; the CodeBuddy chat upstream round-trips it, and a provider that + * does not know the field ignores it. + */ + reasoning?: string; +} + +export interface StreamingToolUseState { + id: string; + name: string; + input: string; + index: number; + started: boolean; + blockEmitted: boolean; +} diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index 97dc6dc..ffccfb7 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -1,1637 +1,26 @@ import type { NextRequest } from 'next/server'; -import { - getDefaultModel, - isWebFetchEnabled, - isWebSearchEnabled, -} from '../domain/config'; import type { DebugTrace } from '../domain/debug'; - -import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; import { - getServerToolStreamEvent, - getServerToolExecutions, - getServerToolTurns, - type ServerToolExecution, - type ServerToolTurn, -} from './web-search-loop'; + anthropicErrorType, + createAnthropicError, + getUpstreamErrorMessage, +} from './anthropic-errors'; import { - anthropicStreamErrorChunks, - createStreamCloser, - toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { stringifyContent } from '../shared/content'; -import { extractErrorMessage } from '../shared/http'; -import { createSseResponse } from '../shared/sse'; + buildChatRequestBody, + shouldBridgeAnthropicServerTools, +} from './anthropic-request'; +import { mapOpenAIResponseToAnthropic } from './anthropic-response'; import { - markServerTool, - normalizeToolName, - WEB_FETCH_TOOL_NAME, - WEB_FETCH_TOOL_TYPE_PREFIX, - WEB_SEARCH_TOOL_NAME, - WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; - -const MAX_STREAM_FRAME_LENGTH = 1_000_000; - -// --------------------------------------------------------------------------- -// Anthropic Messages API types -// --------------------------------------------------------------------------- - -interface AnthropicImageSource { - type?: string; - media_type?: string; - data?: string; - url?: string; -} - -interface AnthropicContentBlock { - type: string; - text?: string; - cache_control?: { type?: string }; - id?: string; - name?: string; - input?: unknown; - thinking?: string; - /** - * Accepted on inbound blocks but never sent by us — see - * `buildThinkingBlock`. Anthropic's signatures hold an encrypted copy of the - * reasoning; a client may replay one from a session it started elsewhere, and - * we skip those rather than forward ciphertext as if it were text. - */ - signature?: string; - /** Present on `redacted_thinking` blocks, which carry no readable text. */ - data?: string; - tool_use_id?: string; - content?: unknown; - source?: AnthropicImageSource; -} - -interface AnthropicMessage { - role: 'user' | 'assistant'; - content: string | AnthropicContentBlock[]; -} - -interface AnthropicTool { - name: string; - description?: string; - input_schema: Record; - type?: string; -} - -interface AnthropicThinkingConfig { - type?: string; - budget_tokens?: number; -} - -interface AnthropicMessagesRequestBody { - model?: string; - messages?: AnthropicMessage[]; - system?: string | AnthropicContentBlock[]; - max_tokens?: number; - temperature?: number; - top_p?: number; - top_k?: number; - stop_sequences?: string[]; - stream?: boolean; - tools?: AnthropicTool[]; - tool_choice?: unknown; - thinking?: AnthropicThinkingConfig; - metadata?: Record; -} - -// --------------------------------------------------------------------------- -// OpenAI response types (mirrors of codebuddy.ts internals) -// --------------------------------------------------------------------------- - -interface OpenAIToolCall { - index?: number; - id?: string; - type?: string; - function?: { - arguments?: string; - name?: string; - }; -} - -interface OpenAIChatMessage { - role?: string; - content?: unknown; - tool_calls?: OpenAIToolCall[]; - reasoning_content?: string; - reasoning?: string; -} - -interface OpenAIChatChoice { - index?: number; - message?: OpenAIChatMessage; - delta?: OpenAIChatMessage; - finish_reason?: string | null; -} - -interface OpenAIStreamError { - error?: { message?: string; status?: number }; -} - -interface OpenAIUsage { - prompt_tokens?: number; - completion_tokens?: number; - total_tokens?: number; - prompt_tokens_details?: { - cached_tokens?: number; - cache_creation_tokens?: number; - }; - completion_tokens_details?: { - reasoning_tokens?: number; - }; -} - -interface OpenAIChatResponse { - id?: string; - model?: string; - choices?: OpenAIChatChoice[]; - usage?: OpenAIUsage; -} - -interface OpenAIStreamChunk { - id?: string; - model?: string; - choices?: OpenAIChatChoice[]; - usage?: OpenAIUsage; -} - -interface ChatTextBlock { - cache_control?: { type?: string }; - text: string; - type: 'text'; -} - -/** - * An image part in the OpenAI Chat shape. Emitted in this shape rather than a - * native Anthropic one because the request is translated to Chat before it - * reaches CodeBuddy: the `chat` upstream forwards it verbatim and the - * `responses` upstream converts it to `input_image`. - */ -interface ChatImageBlock { - cache_control?: { type?: string }; - image_url: { url: string }; - type: 'image_url'; -} - -type ChatContentPart = string | ChatTextBlock | ChatImageBlock; - -type ChatContent = string | Array; - -/** - * Text-only content, used where images are not representable — the system - * prompt and the intermediate text-part buffer. - */ -type ChatTextContent = string | ChatTextBlock[]; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -const createAnthropicId = (prefix: string): string => { - return `${prefix}_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const mapTextPartsToChatContent = ( - parts: Array, -): ChatTextContent => { - const textParts = parts.filter((part) => - typeof part === 'string' ? part.length > 0 : part.text.length > 0, - ); - const hasStructuredText = textParts.some((part) => typeof part !== 'string'); - - if (!hasStructuredText) { - return textParts.join('\n'); - } - - return textParts.flatMap((part, index) => [ - ...(index > 0 ? [{ type: 'text' as const, text: '\n' }] : []), - typeof part === 'string' ? { type: 'text' as const, text: part } : part, - ]); -}; - -/** - * Builds the `image_url` value for an Anthropic image block. Base64 sources - * become a data URI because the upstream Chat/Responses APIs expect a URL; - * `url` sources pass through untouched. Returns undefined for an unusable - * source so the caller can fall back to a text placeholder rather than - * emitting a block the upstream would reject. - */ -const buildChatImageUrl = ( - source: AnthropicImageSource | undefined, -): string | undefined => { - if (!source || typeof source !== 'object') { - return undefined; - } - - if (source.type === 'url' || (!source.data && source.url)) { - return typeof source.url === 'string' && source.url - ? source.url - : undefined; - } - - if (typeof source.data !== 'string' || !source.data) { - return undefined; - } - - const mediaType = - typeof source.media_type === 'string' && source.media_type - ? source.media_type - : 'image/png'; - - return `data:${mediaType};base64,${source.data}`; -}; - -/** - * Like `mapTextPartsToChatContent`, but keeps image parts as real image - * blocks instead of collapsing them into text. Falls back to the text-only - * result when nothing resolved to an image. - */ -const mapContentPartsToChat = (parts: ChatContentPart[]): ChatContent => { - const hasImage = parts.some( - (part) => typeof part === 'object' && part.type === 'image_url', - ); - - if (!hasImage) { - return mapTextPartsToChatContent( - parts.filter( - (part): part is string | ChatTextBlock => - typeof part === 'string' || part.type === 'text', - ), - ); - } - - const blocks: Array = []; - let pendingText: Array = []; - - const flushText = (): void => { - if (!pendingText.length) { - return; - } - const textContent = mapTextPartsToChatContent(pendingText); - if (typeof textContent === 'string') { - blocks.push({ type: 'text', text: textContent }); - } else { - blocks.push(...textContent); - } - pendingText = []; - }; - - for (const part of parts) { - if (typeof part === 'object' && part.type === 'image_url') { - flushText(); - blocks.push(part); - continue; - } - pendingText.push(part); - } - - flushText(); - - return blocks; -}; - -const extractSystemText = ( - system: string | AnthropicContentBlock[] | undefined, -): ChatTextContent => { - if (!system) { - return ''; - } - - if (typeof system === 'string') { - return system; - } - - return mapTextPartsToChatContent( - system.map((block) => { - if (block.type === 'text') { - const text = block.text ?? ''; - - return block.cache_control - ? { type: 'text', text, cache_control: block.cache_control } - : text; - } - - return stringifyContent(block); - }), - ); -}; - -// --------------------------------------------------------------------------- -// Request translation: Anthropic → OpenAI -// --------------------------------------------------------------------------- - -interface ChatMessage { - role: string; - content: ChatContent | null; - tool_calls?: Array<{ - id: string; - type: string; - function: { - name: string; - arguments: string; - }; - }>; - tool_call_id?: string; - /** - * Prior-turn reasoning for this assistant message. Not part of the OpenAI - * schema; the CodeBuddy chat upstream round-trips it, and a provider that - * does not know the field ignores it. - */ - reasoning?: string; -} - -const decodeOpaqueServerToolContent = (value: unknown): unknown => { - if (typeof value !== 'string' || !value) { - return null; - } - - try { - const binary = atob(value); - const bytes = Uint8Array.from(binary, (character) => - character.charCodeAt(0), - ); - - return JSON.parse(new TextDecoder().decode(bytes)) as unknown; - } catch { - return null; - } -}; - -const formatAnthropicServerToolResult = ( - block: AnthropicContentBlock, -): string => { - if (block.type === 'web_search_tool_result' && Array.isArray(block.content)) { - return block.content - .map((value, index) => { - const item = - value && typeof value === 'object' - ? (value as Record) - : {}; - const decoded = decodeOpaqueServerToolContent(item.encrypted_content); - const source = - decoded && typeof decoded === 'object' - ? (decoded as Record) - : item; - const title = String(source.title ?? item.title ?? '').trim(); - const url = String(source.url ?? item.url ?? '').trim(); - const text = String( - source.content ?? source.snippet ?? source.text ?? '', - ).trim(); - - return [ - `${index + 1}. ${title || url || 'Search result'}`, - ...(url ? [`URL: ${url}`] : []), - ...(text ? [text] : []), - ].join('\n'); - }) - .join('\n\n'); - } - - if ( - block.type === 'web_fetch_tool_result' && - block.content && - typeof block.content === 'object' - ) { - const result = block.content as Record; - const document = - result.content && typeof result.content === 'object' - ? (result.content as Record) - : null; - const source = - document?.source && typeof document.source === 'object' - ? (document.source as Record) - : null; - const url = typeof result.url === 'string' ? result.url : ''; - const text = typeof source?.data === 'string' ? source.data : ''; - - return [url, text].filter(Boolean).join('\n\n'); - } - - // Nested images are emitted as real image parts by - // `collectAnthropicNestedImages`, so they are excluded here to keep their - // base64 payload out of the text. - if (Array.isArray(block.content)) { - return stringifyContent( - block.content.filter((value) => { - return !( - value && - typeof value === 'object' && - (value as AnthropicContentBlock).type === 'image' - ); - }), - ); - } - - return typeof block.content === 'string' - ? block.content - : stringifyContent(block.content); -}; - -/** - * Images nested inside a `tool_result` content array, e.g. a screenshot a tool - * returned. The outer block is handled by the `tool_result` branch, whose - * formatter stringifies nested content — so without extracting them here the - * model would receive the base64 payload as text. - */ -const collectAnthropicNestedImages = ( - block: AnthropicContentBlock, -): ChatImageBlock[] => { - if (!Array.isArray(block.content)) { - return []; - } - - return block.content.flatMap((value): ChatImageBlock[] => { - if (!value || typeof value !== 'object') { - return []; - } - - const nested = value as AnthropicContentBlock; - - if (nested.type !== 'image') { - return []; - } - - const imageUrl = buildChatImageUrl(nested.source); - - return imageUrl - ? [{ type: 'image_url', image_url: { url: imageUrl } }] - : []; - }); -}; - -const mapAnthropicContentToChat = ( - content: string | AnthropicContentBlock[], - role: 'user' | 'assistant', -): ChatMessage[] => { - if (typeof content === 'string') { - return [{ role, content }]; - } - - const parts: ChatContentPart[] = []; - const toolCalls: Array<{ - id: string; - type: string; - function: { - name: string; - arguments: string; - }; - }> = []; - const toolResults: ChatMessage[] = []; - const messages: ChatMessage[] = []; - /** - * Reasoning recovered from thinking blocks in this assistant message. - * - * Attached to the message the blocks belong to rather than sent on its own: - * a bare reasoning entry is not a valid chat message, and the upstream needs - * the reasoning alongside the text and tool calls it produced. - */ - let pendingReasoning = ''; - const flushAssistantMessage = (): void => { - const content = mapContentPartsToChat(parts); - const hasContent = typeof content === 'string' ? content.length > 0 : true; - - if (!toolCalls.length && !hasContent && !pendingReasoning) { - return; - } - - messages.push({ - role: 'assistant', - content: hasContent ? content : null, - ...(toolCalls.length ? { tool_calls: [...toolCalls] } : {}), - // `reasoning` is the field the CodeBuddy chat upstream round-trips. It - // is not part of the OpenAI schema, but the upstream accepts it and - // ignoring an unknown field costs nothing if it ever stops doing so. - ...(pendingReasoning ? { reasoning: pendingReasoning } : {}), - }); - parts.length = 0; - toolCalls.length = 0; - pendingReasoning = ''; - }; - - for (const block of content) { - if (block.type === 'text') { - const text = block.text ?? ''; - - parts.push( - block.cache_control - ? { type: 'text', text, cache_control: block.cache_control } - : text, - ); - } else if (block.type === 'tool_use' || block.type === 'server_tool_use') { - toolCalls.push({ - id: block.id ?? createAnthropicId('toolu'), - type: 'function', - function: { - name: block.name ?? 'unknown', - arguments: JSON.stringify(block.input ?? {}), - }, - }); - } else if ( - block.type === 'tool_result' || - block.type === 'web_search_tool_result' || - block.type === 'web_fetch_tool_result' - ) { - const nestedImages = collectAnthropicNestedImages(block); - - const resultMessage: ChatMessage = { - role: 'tool', - content: nestedImages.length - ? mapContentPartsToChat([ - formatAnthropicServerToolResult(block), - ...nestedImages, - ]) - : formatAnthropicServerToolResult(block), - tool_call_id: block.tool_use_id ?? '', - }; - - if (role === 'assistant') { - flushAssistantMessage(); - messages.push(resultMessage); - } else { - toolResults.push(resultMessage); - } - } else if ( - block.type === 'thinking' || - block.type === 'redacted_thinking' - ) { - // Replaying prior-turn reasoning is required inside a tool-use turn and - // harmless elsewhere, so recover it instead of dropping it. - // - // The `thinking` field carries the reasoning. A `signature` is only ever - // read when it is one we minted on the Responses path; a genuine - // Anthropic signature is ciphertext, and forwarding it upstream would put - // gibberish where reasoning belongs. - // - // `redacted_thinking` has no readable text at all, only `data`, but must - // still be matched here: without this branch it fell through to - // `stringifyContent` and the model received a JSON dump of the opaque - // payload as if it were user prose. - const reasoning = block.thinking ?? ''; - - if (reasoning) { - pendingReasoning = pendingReasoning - ? `${pendingReasoning}${reasoning}` - : reasoning; - } - } else if (block.type === 'image' && block.source) { - // Anthropic sends `{ type: 'image', source: { type: 'base64' | 'url', - // media_type, data | url } }`. Emit a real image block so the upstream - // model sees the image; without this branch the block fell through to - // `stringifyContent` and the model received a JSON dump of the base64 - // payload as text. An `image` block with no `source` is not a real - // Anthropic image, so it keeps the generic stringified handling. - const imageUrl = buildChatImageUrl(block.source); - - parts.push( - imageUrl - ? { - type: 'image_url', - image_url: { url: imageUrl }, - // Preserve an explicit cache breakpoint, matching how text - // blocks carry `cache_control` through. Without this the - // requested breakpoint is dropped and `applyPromptCacheControl` - // falls back to its own automatic placement. - ...(block.cache_control - ? { cache_control: block.cache_control } - : {}), - } - : stringifyContent(block), - ); - } else { - parts.push(stringifyContent(block)); - } - } - - if (role === 'user') { - messages.push(...toolResults); - const content = mapContentPartsToChat(parts); - const hasContent = typeof content === 'string' ? content.length > 0 : true; - if (hasContent) { - messages.push({ role: 'user', content }); - } - } else { - flushAssistantMessage(); - } - - return messages; -}; - -const mapAnthropicMessagesToChat = ( - messages: AnthropicMessage[], -): ChatMessage[] => { - const result: ChatMessage[] = []; - - for (const msg of messages) { - const mapped = mapAnthropicContentToChat(msg.content, msg.role); - - if (mapped.length === 0) { - continue; - } - - for (const item of mapped) { - result.push({ - ...item, - }); - } - } - - return result; -}; - -const mapAnthropicToolsToChat = ( - tools: AnthropicTool[] | undefined, -): unknown[] | undefined => { - if (!tools?.length) { - return undefined; - } - - return tools.map((tool) => { - const mapped = { - type: 'function', - function: { - name: tool.name, - description: tool.description, - parameters: tool.input_schema, - }, - }; - const normalizedType = normalizeToolName(tool.type ?? ''); - const serverDeclared = [ - WEB_SEARCH_TOOL_TYPE_PREFIX, - WEB_FETCH_TOOL_TYPE_PREFIX, - ].some((prefix) => normalizedType.startsWith(normalizeToolName(prefix))); - - return serverDeclared ? markServerTool(mapped) : mapped; - }); -}; - -const shouldBridgeAnthropicServerTools = async ( - tools: AnthropicTool[] | undefined, -): Promise => { - const names = new Set( - (tools ?? []).map((tool) => normalizeToolName(tool.name)), - ); - const [searchEnabled, fetchEnabled] = await Promise.all([ - names.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) - ? isWebSearchEnabled() - : false, - names.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) - ? isWebFetchEnabled() - : false, - ]); - - return searchEnabled || fetchEnabled; -}; - -const mapAnthropicToolChoiceToChat = (toolChoice: unknown): unknown => { - if (!toolChoice || typeof toolChoice !== 'object') { - return toolChoice; - } - - const tc = toolChoice as { type?: string; name?: string }; - - if (tc.type === 'auto') { - return 'auto'; - } - - if (tc.type === 'any') { - return 'required'; - } - - if (tc.type === 'tool' && tc.name) { - return { - type: 'function', - function: { name: tc.name }, - }; - } - - if (tc.type === 'none') { - return 'none'; - } - - return toolChoice; -}; - -const buildChatRequestBody = async ( - body: AnthropicMessagesRequestBody, -): Promise> => { - const systemText = extractSystemText(body.system); - const chatMessages = mapAnthropicMessagesToChat(body.messages ?? []); - - const messages: ChatMessage[] = []; - const disableParallelToolUse = - body.tool_choice && typeof body.tool_choice === 'object' - ? (body.tool_choice as { disable_parallel_tool_use?: unknown }) - .disable_parallel_tool_use - : undefined; - - if (systemText) { - messages.push({ role: 'system', content: systemText }); - } - - messages.push(...chatMessages); - - const result: Record = { - model: - typeof body.model === 'string' && body.model.trim() - ? body.model - : await getDefaultModel('claude-sonnet-4.6'), - messages, - stream: body.stream ?? false, - max_tokens: body.max_tokens, - temperature: body.temperature, - top_p: body.top_p, - stop: body.stop_sequences, - tools: mapAnthropicToolsToChat(body.tools), - tool_choice: mapAnthropicToolChoiceToChat(body.tool_choice), - parallel_tool_calls: - typeof disableParallelToolUse === 'boolean' - ? !disableParallelToolUse - : undefined, - }; - - // Pass through thinking/reasoning config so upstream models that support - // extended thinking can honor it. - if (body.thinking) { - result.thinking = body.thinking; - } - - return result; -}; - -// --------------------------------------------------------------------------- -// Response translation: OpenAI → Anthropic (non-streaming) -// --------------------------------------------------------------------------- - -const mapOpenAIUsageToAnthropic = ( - usage: OpenAIUsage | undefined, - serverToolExecutions: ServerToolExecution[] = [], -): Record => { - const cacheCreationTokens = - usage?.prompt_tokens_details?.cache_creation_tokens ?? 0; - const cacheReadTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; - // prompt_tokens is the total prompt count including cached tokens. - // Anthropic reports cached/created tokens separately, so input_tokens - // must be the non-cache remainder to avoid double-counting. - const inputTokens = Math.max( - 0, - (usage?.prompt_tokens ?? 0) - cacheCreationTokens - cacheReadTokens, - ); - const outputTokens = usage?.completion_tokens ?? 0; - - const mapped: Record> = { - input_tokens: inputTokens, - output_tokens: outputTokens, - cache_creation_input_tokens: cacheCreationTokens, - cache_read_input_tokens: cacheReadTokens, - }; - - if (serverToolExecutions.length) { - mapped.server_tool_use = { - web_search_requests: serverToolExecutions.filter( - (execution) => execution.type === 'web_search', - ).length, - web_fetch_requests: serverToolExecutions.filter( - (execution) => execution.type === 'web_fetch', - ).length, - }; - } - - return mapped; -}; - -const encodeOpaqueServerToolContent = (value: unknown): string => { - const bytes = new TextEncoder().encode(JSON.stringify(value)); - let binary = ''; - - for (const byte of bytes) { - binary += String.fromCharCode(byte); - } - - return btoa(binary); -}; - -const buildAnthropicServerToolBlocks = ( - execution: ServerToolExecution, -): AnthropicContentBlock[] => { - const id = createAnthropicId('srvtoolu'); - const result = - execution.type === 'web_search' - ? { - type: 'web_search_tool_result', - tool_use_id: id, - content: execution.result.results.map((item) => ({ - type: 'web_search_result', - url: item.url ?? '', - title: item.title ?? '', - encrypted_content: encodeOpaqueServerToolContent(item), - })), - } - : { - type: 'web_fetch_tool_result', - tool_use_id: id, - content: { - type: 'web_fetch_result', - url: execution.result.url ?? execution.input.url, - content: { - type: 'document', - source: { - type: 'text', - media_type: 'text/plain', - data: execution.result.content, - }, - }, - }, - }; - - return [ - { - type: 'server_tool_use', - id, - name: execution.type, - input: execution.input, - }, - result, - ]; -}; - -const buildAllAnthropicServerToolBlocks = ( - executions: ServerToolExecution[], -): AnthropicContentBlock[] => - executions.flatMap(buildAnthropicServerToolBlocks); - -/** - * We do not mint a `signature` on this path. It would have to duplicate the - * `thinking` text to be replayable, which puts the reasoning on the wire twice - * for callers that count it — and the block already replays fine: Anthropic - * clients echo `thinking` back, which is what inbound handling reads. - */ -const buildThinkingBlock = (thinking: string): AnthropicContentBlock => ({ - type: 'thinking', - thinking, -}); - -/** - * Lays a server-tool turn out the way Anthropic does: each hop contributes its - * own thinking and text, followed by the tool blocks that hop triggered. - * - * `turns` carries the per-hop grouping the OpenAI-shaped payload cannot. Under - * that protocol a multi-hop turn collapses into one `content` string and one - * `reasoning_content` string, which loses where one hop's reasoning ends and the - * next begins — so the grouping has to be recovered before it is joined, which - * is why the loop emits it alongside the strings rather than this file - * reconstructing it. - * - * Anthropic's own server tools run multiple hops inside one assistant message, - * and a client replaying that message expects `[thinking] [text] [tool_use] - * [tool_result] [thinking] [text]`. Gathering the blocks by kind instead — every - * tool ahead of all the prose — puts each search before the reasoning that asked - * for it and merges hops that were never contiguous. - */ -const buildAnthropicTurnBlocks = ( - turns: ServerToolTurn[], -): AnthropicContentBlock[] => { - const blocks: AnthropicContentBlock[] = []; - - turns.forEach((turn) => { - if (turn.reasoning) { - blocks.push(buildThinkingBlock(turn.reasoning)); - } - - if (turn.text) { - blocks.push({ type: 'text', text: turn.text }); - } - - blocks.push(...buildAllAnthropicServerToolBlocks(turn.executions)); - }); - - return blocks; -}; - -const mapOpenAIResponseToAnthropic = ( - openaiResponse: OpenAIChatResponse, - model: string, - serverToolExecutions: ServerToolExecution[] = [], - turns?: ServerToolTurn[], -): Record => { - const choice = openaiResponse.choices?.[0]; - const message = choice?.message; - - // Thinking / reasoning content - const reasoningText = message?.reasoning_content ?? message?.reasoning ?? ''; - - // Text content - const textContent = - typeof message?.content === 'string' ? message.content : ''; - - // With per-hop grouping the turns already hold every block in order, prose - // included. Without it — no server tool ran, or a path that never grouped the - // hops — fall back to Anthropic's own order: thinking and text first, then - // the server-tool blocks they led to. - const contentBlocks: AnthropicContentBlock[] = turns - ? buildAnthropicTurnBlocks(turns) - : []; - - if (!turns) { - if (reasoningText) { - contentBlocks.push(buildThinkingBlock(reasoningText)); - } - - if (textContent) { - contentBlocks.push({ type: 'text', text: textContent }); - } - - contentBlocks.push( - ...buildAllAnthropicServerToolBlocks(serverToolExecutions), - ); - } - - // Tool calls - const toolCalls = message?.tool_calls ?? []; - - for (const call of toolCalls) { - let input: unknown = {}; - - try { - input = JSON.parse(call.function?.arguments ?? '{}'); - } catch { - input = {}; - } - - contentBlocks.push({ - type: 'tool_use', - id: call.id ?? createAnthropicId('toolu'), - name: call.function?.name ?? 'unknown', - input, - }); - } - - const stopReason = mapFinishReasonToAnthropic( - choice?.finish_reason, - toolCalls.length > 0, - ); - - return { - id: openaiResponse.id ?? createAnthropicId('msg'), - type: 'message', - role: 'assistant', - model, - content: contentBlocks, - stop_reason: stopReason, - stop_sequence: null, - usage: mapOpenAIUsageToAnthropic( - openaiResponse.usage, - serverToolExecutions, - ), - }; -}; - -const mapFinishReasonToAnthropic = ( - finishReason: string | null | undefined, - hasToolCalls: boolean, -): string => { - if (hasToolCalls || finishReason === 'tool_calls') { - return 'tool_use'; - } - - if (finishReason === 'length') { - return 'max_tokens'; - } - - if (finishReason === 'stop' || !finishReason) { - return 'end_turn'; - } - - return 'end_turn'; -}; - -// --------------------------------------------------------------------------- -// Response translation: OpenAI SSE → Anthropic SSE (streaming) -// --------------------------------------------------------------------------- - -interface StreamingToolUseState { - id: string; - name: string; - input: string; - index: number; - started: boolean; - blockEmitted: boolean; -} - -const mapOpenAIStreamToAnthropicSSE = ( - upstreamResponse: Response, - model: string, - options?: { - emitMessageStart?: boolean; - initialContentBlockCount?: number; - messageId?: string; - serverToolExecutions?: ServerToolExecution[]; - }, -): Response => { - if (!upstreamResponse.body) { - return createSseResponse(null, { status: upstreamResponse.status }); - } - - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - - const messageId = options?.messageId ?? createAnthropicId('msg'); - const serverToolExecutions = - options?.serverToolExecutions ?? getServerToolExecutions(upstreamResponse); - const serverToolUseIds = new Map(); - const toolUseStates = new Map(); - let nextToolIndex = 0; - let started = options?.emitMessageStart === false; - let thinkingStarted = false; - let thinkingBlockIndex = -1; - let textStarted = false; - let textBlockIndex = -1; - // Tracks how many content blocks (thinking + text) have been opened - // so tool_use blocks get correct sequential indices even after the - // prior blocks are closed mid-stream. - let contentBlockCount = options?.initialContentBlockCount ?? 0; - let finishReason: string | null = null; - let hasToolCalls = false; - let usage: OpenAIUsage | undefined; - - const enqueueEvent = (event: Record): void => { - controller.enqueue( - encoder.encode( - `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, - ), - ); - }; - - let controller: ReadableStreamDefaultController; - - // Close any open text/thinking block before starting a tool_use block. - // Anthropic streaming requires each block to be stopped before the next. - const closeOpenTextBlocks = (): void => { - if (thinkingStarted) { - // Anthropic emits the signature last, just before the block closes — but - // we do not send one here: the reasoning already went out as - // `thinking_delta`s, and duplicating it into a signature would put the - // text on the wire twice for callers that count it. See - // `buildThinkingBlock`. - enqueueEvent({ - type: 'content_block_stop', - index: thinkingBlockIndex, - }); - thinkingStarted = false; - } - - if (textStarted) { - enqueueEvent({ - type: 'content_block_stop', - index: textBlockIndex, - }); - textStarted = false; - } - }; - - const processChunk = (chunk: OpenAIStreamChunk): void => { - if (!started) { - started = true; - enqueueEvent({ - type: 'message_start', - message: { - id: messageId, - type: 'message', - role: 'assistant', - content: [], - model, - stop_reason: null, - stop_sequence: null, - usage: { - input_tokens: chunk.usage?.prompt_tokens ?? 0, - output_tokens: 0, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, - }, - }, - }); - } - - if (chunk.usage) { - usage = chunk.usage; - } - - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - - if (!delta) { - return; - } - - // Reasoning / thinking content - const reasoningText = delta.reasoning_content ?? delta.reasoning ?? ''; - - if (reasoningText) { - if (!thinkingStarted) { - thinkingBlockIndex = contentBlockCount; - thinkingStarted = true; - enqueueEvent({ - type: 'content_block_start', - index: thinkingBlockIndex, - content_block: { - type: 'thinking', - thinking: '', - }, - }); - contentBlockCount++; - } - - enqueueEvent({ - type: 'content_block_delta', - index: thinkingBlockIndex, - delta: { - type: 'thinking_delta', - thinking: reasoningText, - }, - }); - } - - // Text content - if (delta.content) { - if (!textStarted) { - // Close the thinking block before starting text so Anthropic - // stream consumers see properly ordered, non-overlapping blocks. - closeOpenTextBlocks(); - - textBlockIndex = contentBlockCount; - textStarted = true; - enqueueEvent({ - type: 'content_block_start', - index: textBlockIndex, - content_block: { - type: 'text', - text: '', - }, - }); - contentBlockCount++; - } - - enqueueEvent({ - type: 'content_block_delta', - index: textBlockIndex, - delta: { - type: 'text_delta', - text: delta.content, - }, - }); - } - - // Tool calls - if (delta.tool_calls?.length) { - hasToolCalls = true; - - // Anthropic streaming requires each content block to be closed - // before the next one starts. If we already opened a text or - // thinking block, close it now so the tool_use block is well-formed. - closeOpenTextBlocks(); - - for (const call of delta.tool_calls) { - const callId = call.id ?? `toolu_${nextToolIndex}`; - const key = callId; - - if (!toolUseStates.has(key)) { - const blockIndex = contentBlockCount + nextToolIndex; - - toolUseStates.set(key, { - id: callId, - name: '', - input: '', - index: blockIndex, - started: false, - blockEmitted: false, - }); - nextToolIndex++; - } - - const state = toolUseStates.get(key)!; - - // Accumulate name fragments (upstream may stream the function - // name across multiple deltas, e.g. "look" + "up"). - if (call.function?.name) { - state.name += call.function.name; - } - - // Emit content_block_start lazily — once we have a name and at - // least one arguments fragment, so the block header carries the - // full tool name instead of a partial fragment. - if (!state.blockEmitted && state.name && call.function?.arguments) { - state.blockEmitted = true; - enqueueEvent({ - type: 'content_block_start', - index: state.index, - content_block: { - type: 'tool_use', - id: state.id, - name: state.name, - input: {}, - }, - }); - } - - if (call.function?.arguments) { - state.input += call.function.arguments; - enqueueEvent({ - type: 'content_block_delta', - index: state.index, - delta: { - type: 'input_json_delta', - partial_json: call.function.arguments, - }, - }); - } - } - } - - if (choice?.finish_reason) { - finishReason = choice.finish_reason; - } - }; - - const finalize = (): void => { - // Close any remaining open text/thinking blocks. - closeOpenTextBlocks(); - - // Close tool use blocks - for (const [, state] of toolUseStates) { - // If the block start was never emitted (e.g. name-only deltas - // with no arguments), emit it now so the block is well-formed. - if (!state.blockEmitted) { - state.blockEmitted = true; - enqueueEvent({ - type: 'content_block_start', - index: state.index, - content_block: { - type: 'tool_use', - id: state.id, - name: state.name || 'unknown', - input: {}, - }, - }); - } - - enqueueEvent({ - type: 'content_block_stop', - index: state.index, - }); - } - - const stopReason = mapFinishReasonToAnthropic(finishReason, hasToolCalls); - - enqueueEvent({ - type: 'message_delta', - delta: { - stop_reason: stopReason, - stop_sequence: null, - }, - usage: mapOpenAIUsageToAnthropic(usage, serverToolExecutions), - }); - - enqueueEvent({ - type: 'message_stop', - }); - }; - - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - let streamRejected = false; - const closer = createStreamCloser(); - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - const stream = new ReadableStream({ - start: (ctrl) => { - controller = ctrl; - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - const rejectStream = ( - message = 'Upstream SSE frame exceeds the maximum size', - status?: number, - ): void => { - streamRejected = true; - enqueueEvent({ - type: 'error', - error: { - // An upstream status names the failure precisely, so it decides - // the type: 429 has to arrive as `rate_limit_error` or a client - // that retries on that type alone stops retrying an exhausted - // quota. Without one, fall back to the message: an oversized frame - // is a malformed stream (`invalid_request_error`), while an - // upstream deadline is the server failing (`api_error`, the type - // clients treat as retryable). - type: - typeof status === 'number' - ? anthropicErrorType(status) - : message.includes('did not produce output') - ? 'api_error' - : 'invalid_request_error', - message, - }, - }); - }; - - const flushFrames = (frames: string[]): void => { - for (const frame of frames) { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - rejectStream(); - return; - } - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - continue; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - try { - const chunk = JSON.parse(raw) as OpenAIStreamChunk; - const serverToolEvent = getServerToolStreamEvent(chunk); - - if (serverToolEvent?.phase === 'call') { - closeOpenTextBlocks(); - const index = contentBlockCount++; - const toolUseId = createAnthropicId('srvtoolu'); - serverToolUseIds.set(serverToolEvent.invocation.id, toolUseId); - enqueueEvent({ - type: 'content_block_start', - index, - content_block: { - type: 'server_tool_use', - id: toolUseId, - name: serverToolEvent.invocation.type, - input: {}, - }, - }); - enqueueEvent({ - type: 'content_block_delta', - index, - delta: { - type: 'input_json_delta', - partial_json: JSON.stringify( - serverToolEvent.invocation.input, - ), - }, - }); - enqueueEvent({ type: 'content_block_stop', index }); - continue; - } - - if (serverToolEvent?.phase === 'result') { - closeOpenTextBlocks(); - serverToolExecutions.push(serverToolEvent.execution); - const index = contentBlockCount++; - const resultBlock = buildAnthropicServerToolBlocks( - serverToolEvent.execution, - )[1]; - enqueueEvent({ - type: 'content_block_start', - index, - content_block: { - ...resultBlock, - tool_use_id: serverToolUseIds.get( - serverToolEvent.execution.id, - ), - }, - }); - enqueueEvent({ type: 'content_block_stop', index }); - continue; - } - - const upstreamError = chunk as OpenAIStreamError; - if (upstreamError.error?.message) { - rejectStream( - upstreamError.error.message, - upstreamError.error.status, - ); - return; - } - processChunk(chunk); - } catch { - // Skip unparseable frames - } - } - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - if (buffer.trim()) { - flushFrames([buffer]); - } - if (!streamRejected) { - finalize(); - } - releaseReader(); - controller.close(); - return; - } - - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - rejectStream(); - } else { - flushFrames(frames); - } - if (streamRejected) { - try { - await reader!.cancel(); - } finally { - releaseReader(); - controller.close(); - } - return; - } - } - }; - - void pump().catch((error) => { - if (cancelled) return; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage === null) { - closer.mark(); - controller.error(error); - return; - } - - void reader?.cancel().then( - () => undefined, - () => undefined, - ); - releaseReader(); - closer.fail(controller, anthropicStreamErrorChunks(timeoutMessage)); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - releaseReader(); - } - }, - }); - - return createSseResponse(stream, { status: 200 }); -}; - -const createAnthropicServerToolEventStream = ( - request: NextRequest, - chatBody: Record, - model: string, - debugTrace?: DebugTrace, -): Response => { - const encoder = new TextEncoder(); - const messageId = createAnthropicId('msg'); - let activeReader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - - const stream = new ReadableStream({ - start: (controller) => { - const enqueueEvent = (event: Record): void => { - if (cancelled) return; - controller.enqueue( - encoder.encode( - `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`, - ), - ); - }; - - enqueueEvent({ - type: 'message_start', - message: { - id: messageId, - type: 'message', - role: 'assistant', - content: [], - model, - stop_reason: null, - stop_sequence: null, - usage: { - input_tokens: 0, - output_tokens: 0, - cache_creation_input_tokens: 0, - cache_read_input_tokens: 0, - }, - }, - }); - - const run = async (): Promise => { - const upstreamResponse = await proxyChatCompletions( - request, - chatBody as ChatRequestBody, - undefined, - debugTrace, - '/v1/messages', - // The findings reach the client as `web_search_tool_result` blocks, - // so the loop must not also fold them into the assistant text. - { emitStreamEvents: true, findingsAsStructuredBlocks: true }, - ); - - if (cancelled) { - await upstreamResponse.body?.cancel(); - return; - } - - if (!upstreamResponse.ok || !upstreamResponse.body) { - // A rate limit has to arrive as `rate_limit_error`, or a client that - // retries on that type alone will treat an exhausted quota as a - // generic failure and stop retrying — so the upstream status drives - // the event type even though the envelope is already streaming and - // the HTTP status cannot be changed. - const message = upstreamResponse.ok - ? 'Upstream request failed' - : await getUpstreamErrorMessage(upstreamResponse).catch( - () => 'Upstream request failed', - ); - - enqueueEvent({ - type: 'error', - error: { - type: anthropicErrorType(upstreamResponse.status), - message, - }, - }); - controller.close(); - return; - } - - const mappedResponse = mapOpenAIStreamToAnthropicSSE( - upstreamResponse, - model, - { - emitMessageStart: false, - initialContentBlockCount: 0, - messageId, - serverToolExecutions: [], - }, - ); - const reader = mappedResponse.body!.getReader(); - activeReader = reader; - - while (true) { - const { done, value } = await reader.read(); - if (cancelled) return; - if (done) break; - controller.enqueue(value); - } - - reader.releaseLock(); - activeReader = null; - controller.close(); - }; - - void run().catch((error) => { - if (!cancelled) controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - await activeReader?.cancel(reason); - activeReader?.releaseLock(); - activeReader = null; - }, - }); - - return createSseResponse(stream); -}; - -const getUpstreamErrorMessage = async (response: Response): Promise => { - const text = await response.text(); - if (!text) return 'Upstream CodeBuddy request failed'; - - try { - return extractErrorMessage(JSON.parse(text) as unknown) ?? text; - } catch { - return text; - } -}; + createAnthropicServerToolEventStream, + mapOpenAIStreamToAnthropicSSE, +} from './anthropic-stream'; +import type { + AnthropicMessagesRequestBody, + OpenAIChatResponse, +} from './anthropic-types'; +import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; +import { getServerToolExecutions, getServerToolTurns } from './web-search-loop'; // --------------------------------------------------------------------------- // Main handler @@ -1698,37 +87,6 @@ export const handleMessagesRequest = async ( } }; -export const anthropicErrorType = (status: number): string => - status === 401 - ? 'authentication_error' - : status === 403 - ? 'permission_error' - : status === 404 - ? 'not_found_error' - : status === 413 - ? 'request_too_large' - : status === 429 - ? 'rate_limit_error' - : status === 529 - ? 'overloaded_error' - : status >= 500 - ? 'api_error' - : 'invalid_request_error'; - -export const createAnthropicError = ( - status: number, - message: string, -): Response => { - const type = anthropicErrorType(status); - - return Response.json( - { - type: 'error', - error: { - type, - message, - }, - }, - { status }, - ); -}; +// Re-exported for the importers that reached these through this module before +// the split: `app/v1/messages/route.ts` and the test suite. +export { anthropicErrorType, createAnthropicError }; From 4982081b50a221f7040656d21504868642268f34 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 22:00:32 +0800 Subject: [PATCH 4/7] refactor(responses): split the Responses handler into focused modules responses.ts was 2961 lines. Session persistence, tool translation, transcript construction, payload assembly and the SSE mapper are now separate modules; the entry point keeps only request dispatch and the three public exports. The id helpers move to responses-ids.ts so the transcript layer can mint an id without reaching back into the payload builder, which would otherwise make the two modules import each other. Also splits the streaming orchestrator out of the SSE mapper, so the file holding the mapper is 727 lines instead of 1016. --- lib/server/proxy/responses-event-stream.ts | 327 +++ lib/server/proxy/responses-ids.ts | 40 + lib/server/proxy/responses-payload.ts | 357 +++ lib/server/proxy/responses-session.ts | 230 ++ lib/server/proxy/responses-stream.ts | 713 +++++ lib/server/proxy/responses-tools.ts | 582 +++++ lib/server/proxy/responses-transcript.ts | 459 ++++ lib/server/proxy/responses-types.ts | 166 ++ lib/server/proxy/responses.ts | 2762 +------------------- 9 files changed, 2918 insertions(+), 2718 deletions(-) create mode 100644 lib/server/proxy/responses-event-stream.ts create mode 100644 lib/server/proxy/responses-ids.ts create mode 100644 lib/server/proxy/responses-payload.ts create mode 100644 lib/server/proxy/responses-session.ts create mode 100644 lib/server/proxy/responses-stream.ts create mode 100644 lib/server/proxy/responses-tools.ts create mode 100644 lib/server/proxy/responses-transcript.ts create mode 100644 lib/server/proxy/responses-types.ts diff --git a/lib/server/proxy/responses-event-stream.ts b/lib/server/proxy/responses-event-stream.ts new file mode 100644 index 0000000..40fe131 --- /dev/null +++ b/lib/server/proxy/responses-event-stream.ts @@ -0,0 +1,327 @@ +// --------------------------------------------------------------------------- +// Responses streaming orchestration +// +// Chooses how a Responses request is served — delegated to the image +// generation loop, bridged as a live server-tool stream, or mapped one chat +// SSE chunk at a time — and owns the session persistence for the result. +// --------------------------------------------------------------------------- + +import type { NextRequest } from 'next/server'; + +import { isWebFetchEnabled, isWebSearchEnabled } from '../domain/config'; +import type { DebugTrace } from '../domain/debug'; +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../search/tool'; +import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { proxyChatCompletions, type ProxyContext } from './codebuddy'; +import { executeImageGenerationLoop } from './image-generation'; +import { + buildResponsesWebSearchCallItem, + mapChatResponseToResponsesStream, +} from './responses-payload'; +import { createResponseId } from './responses-ids'; + +import { mapChatStreamToResponsesEventStream } from './responses-stream'; +import { + hasImageGenerationTool, + normalizeTranscriptMessageToolNames, + translateResponsesToolsToChat, + translateResponsesToolChoiceToChatWithTools, +} from './responses-tools'; +import type { + ResponsesServerToolItem, + ResponseSessionDefaults, + TranscriptMessage, +} from './responses-types'; + +export const createResponsesEventStream = async ( + request: NextRequest, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + maxOutputTokens: number | undefined, + proxyContext: ProxyContext, + debugTrace?: DebugTrace, +): Promise => { + const translatedTools = translateResponsesToolsToChat(defaults.tools); + const translatedToolNames = new Set( + ( + (translatedTools ?? []) as Array<{ + function: { name: string }; + }> + ).map((tool) => normalizeToolName(tool.function.name)), + ); + const [searchEnabled, fetchEnabled] = await Promise.all([ + translatedToolNames.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) + ? isWebSearchEnabled() + : false, + translatedToolNames.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) + ? isWebFetchEnabled() + : false, + ]); + + const chatBody = { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }; + + // Image generation is executed locally, so a streaming request has to be + // buffered first to see whether the model asked for an image. Without this + // the call is forwarded as an ordinary function_call the client is expected + // to resolve — and nothing would ever generate the image. + // + // Handled before the server-tool branch below: a turn may declare both, and + // gating on search/fetch would silently skip generation whenever those were + // enabled. + if (hasImageGenerationTool(defaults.tools)) { + const { executions, response, serverToolExecutions } = + await executeImageGenerationLoop({ + body: chatBody, + // Buffered so the tool call can be inspected before any delta reaches + // the client; the ordinary path below stays live. + callUpstream: (loopBody) => + proxyChatCompletions( + request, + { ...loopBody, stream: false } as never, + proxyContext, + debugTrace, + '/v1/responses', + ), + context: proxyContext, + request, + }); + + // Always consumed, even when nothing was generated: the loop has already + // sent the turn upstream, and re-issuing it would bill twice and could + // return a different answer than the one inspected. + if (!response.ok) { + return response; + } + + return mapChatResponseToResponsesStream( + (await response.json()) as Record, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + executions, + serverToolExecutions, + ); + } + + if (!searchEnabled && !fetchEnabled) { + const upstreamResponse = await proxyChatCompletions( + request, + chatBody as never, + proxyContext, + debugTrace, + '/v1/responses', + ); + + return mapChatStreamToResponsesEventStream( + upstreamResponse, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + ); + } + + const encoder = new TextEncoder(); + const responseId = createResponseId(); + const serverToolItems: ResponsesServerToolItem[] = []; + const itemsByInvocationId = new Map(); + let nextOutputIndex = 0; + const allocateOutputIndex = (): number => nextOutputIndex++; + let activeReader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + + const stream = new ReadableStream({ + start: (controller) => { + const enqueueEvent = ( + payload: Record & { type: string }, + ): void => { + if (cancelled) return; + controller.enqueue( + encoder.encode( + `event: ${payload.type}\ndata: ${JSON.stringify(payload)}\n\n`, + ), + ); + }; + + enqueueEvent({ + type: 'response.created', + response: { + id: responseId, + object: 'response', + created_at: Math.floor(Date.now() / 1000), + model, + output: [], + }, + }); + enqueueEvent({ + type: 'response.in_progress', + response: { id: responseId, status: 'in_progress' }, + }); + + const run = async (): Promise => { + const upstreamResponse = await proxyChatCompletions( + request, + { + model, + messages: [ + ...(defaults.instructions + ? [{ role: 'system', content: defaults.instructions }] + : []), + ...normalizeTranscriptMessageToolNames( + transcript, + defaults.tools, + ), + ], + max_tokens: maxOutputTokens, + stream: true, + tools: translatedTools, + tool_choice: translateResponsesToolChoiceToChatWithTools( + defaults.tools, + defaults.tool_choice, + ), + }, + proxyContext, + debugTrace, + '/v1/responses', + { + emitStreamEvents: true, + onCall: (invocation) => { + const outputIndex = allocateOutputIndex(); + const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; + const item = { + completed: buildResponsesWebSearchCallItem( + invocation, + 'completed', + id, + ), + inProgress: buildResponsesWebSearchCallItem( + invocation, + 'in_progress', + id, + ), + outputIndex, + }; + serverToolItems.push(item); + itemsByInvocationId.set(invocation.id, item); + enqueueEvent({ + type: 'response.output_item.added', + item: item.inProgress, + output_index: outputIndex, + response_id: responseId, + }); + enqueueEvent({ + type: 'response.web_search_call.in_progress', + item_id: id, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.web_search_call.searching', + item_id: id, + output_index: outputIndex, + }); + }, + onResult: (execution) => { + const item = itemsByInvocationId.get(execution.id)!; + const id = String(item.inProgress.id); + item.completed = buildResponsesWebSearchCallItem( + execution, + 'completed', + id, + ); + enqueueEvent({ + type: 'response.web_search_call.completed', + item_id: id, + output_index: item.outputIndex, + }); + enqueueEvent({ + type: 'response.output_item.done', + item: item.completed, + output_index: item.outputIndex, + response_id: responseId, + }); + }, + }, + ); + + if (cancelled) { + await upstreamResponse.body?.cancel(); + return; + } + + if (!upstreamResponse.ok || !upstreamResponse.body) { + enqueueEvent({ + type: 'response.error', + error: { message: 'Upstream request failed' }, + }); + controller.enqueue(encodeDoneFrame()); + controller.close(); + return; + } + + const mappedResponse = mapChatStreamToResponsesEventStream( + upstreamResponse, + defaults, + transcript, + model, + previousResponseId, + proxyContext, + responseId, + serverToolItems, + false, + false, + allocateOutputIndex, + true, + ); + const reader = mappedResponse.body!.getReader(); + activeReader = reader; + + while (true) { + const { done, value } = await reader.read(); + if (cancelled) return; + if (done) break; + controller.enqueue(value); + } + + reader.releaseLock(); + activeReader = null; + controller.close(); + }; + + void run().catch((error) => { + if (!cancelled) controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + await activeReader?.cancel(reason); + activeReader?.releaseLock(); + activeReader = null; + }, + }); + + return createSseResponse(stream); +}; diff --git a/lib/server/proxy/responses-ids.ts b/lib/server/proxy/responses-ids.ts new file mode 100644 index 0000000..52150ac --- /dev/null +++ b/lib/server/proxy/responses-ids.ts @@ -0,0 +1,40 @@ +/** + * Identifier minting for Responses objects. + * + * Every id is a short protocol prefix plus a UUID, which is what the Responses + * API expects and what clients key their own bookkeeping off. They live apart + * from the payload builder because the transcript layer needs them too, and + * reaching back into the payload for an id would make the two circular. + */ + +export const createResponseId = (): string => { + return `resp_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const createMessageId = (): string => { + return `msg_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const createResponseReasoningId = (): string => { + return `rs_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +export const createResponseOutputId = (): string => { + return `fc_${crypto.randomUUID().replaceAll('-', '')}`; +}; + +/** + * Rewrites an Anthropic-style tool-call id into the `call_` shape OpenAI + * clients expect. An id the upstream already minted is returned untouched, so + * only synthetic ids are normalised. + */ +export const normalizeToolCallId = ( + id: string | undefined, + index: number, +): string => { + if (id && !id.startsWith('tooluse_')) { + return id; + } + + return `call_${id?.replace(/^tooluse_/, '') ?? index + 1}`; +}; diff --git a/lib/server/proxy/responses-payload.ts b/lib/server/proxy/responses-payload.ts new file mode 100644 index 0000000..e0bf4be --- /dev/null +++ b/lib/server/proxy/responses-payload.ts @@ -0,0 +1,357 @@ +// --------------------------------------------------------------------------- +// Non-streaming Responses payload assembly +// --------------------------------------------------------------------------- + +import { stringifyContent } from '../shared/content'; +import { + createSseResponse, + DONE_FRAME_TEXT, + eventFrameText, +} from '../shared/sse'; +import type { ProxyContext } from './codebuddy'; +import { + buildResponsesImageGenerationCallItem, + type ImageGenerationExecution, +} from './image-generation'; +import { + createMessageId, + createResponseId, + createResponseOutputId, + createResponseReasoningId, + normalizeToolCallId, +} from './responses-ids'; +import { storeResponseSession } from './responses-session'; +import { buildResponsesToolCallOutputItem } from './responses-tools'; +import { + buildAssistantTranscriptToolCalls, + getAssistantTranscriptContent, + mapChatUsageToResponses, + REASONING_PREFIX, +} from './responses-transcript'; +import type { + ChatResponseMessage, + ResponseSessionDefaults, + TranscriptMessage, +} from './responses-types'; +import type { + ServerToolExecution, + ServerToolInvocation, +} from './web-search-loop'; + +export const mapChatResponseToResponsesPayload = async ( + accessKeyId: string | null, + credentialFilename: string | null, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + upstreamPayload: Record, + serverToolExecutions: ServerToolExecution[], + imageExecutions: ImageGenerationExecution[] = [], +): Promise> => { + const responseId = createResponseId(); + const choices = Array.isArray(upstreamPayload.choices) + ? upstreamPayload.choices + : []; + const firstChoice = (choices[0] ?? {}) as { + message?: ChatResponseMessage; + }; + const toolCalls = Array.isArray(firstChoice.message?.tool_calls) + ? firstChoice.message.tool_calls + : []; + const outputText = stringifyContent(firstChoice.message?.content); + const createdAt = Math.floor(Date.now() / 1000); + const output: Array> = [ + ...serverToolExecutions.map((execution) => + buildResponsesWebSearchCallItem(execution, 'completed'), + ), + // Image generation is executed locally, so the standard + // `image_generation_call` item has to be synthesized here — the chat + // upstream has no notion of it. + ...imageExecutions.map((execution) => + buildResponsesImageGenerationCallItem(execution), + ), + ]; + const transcriptToolCalls = buildAssistantTranscriptToolCalls( + toolCalls, + defaults.tools, + ); + + // Emit the reasoning as its own item, ahead of the message it produced. + // + // Clients replay `output` verbatim on the next turn, so this is what lets a + // stateless Responses client carry reasoning forward. Without it the only + // reasoning we ever hand back is a transient `reasoning_text.delta`, which no + // client can replay because it has no id and no blob to send back. + // + // `encrypted_content` holds the reasoning verbatim, not ciphertext. Codex + // never opens it — it only echoes it — so plaintext round-trips exactly as + // well, and encrypting would obscure a value that carries no secret: the + // upstream gave us a summary, and the `summary` field below already shows it. + const reasoningText = + firstChoice.message?.reasoning_content ?? + firstChoice.message?.reasoning ?? + ''; + + if (reasoningText) { + output.push({ + id: createResponseReasoningId(), + type: 'reasoning', + summary: [{ type: 'summary_text', text: reasoningText }], + encrypted_content: `${REASONING_PREFIX}${reasoningText}`, + status: 'completed', + }); + } + + if (outputText || !toolCalls.length) { + output.push({ + id: createMessageId(), + type: 'message', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text: outputText, + annotations: [], + }, + ], + }); + } + + toolCalls.forEach((toolCall, index) => { + output.push( + buildResponsesToolCallOutputItem(defaults.tools, { + arguments: toolCall.function?.arguments ?? '', + callId: normalizeToolCallId(toolCall.id, index), + id: createResponseOutputId(), + name: toolCall.function?.name ?? 'function', + status: 'completed', + }), + ); + }); + + await storeResponseSession({ + accessKeyId, + credentialFilename, + createdAt: Date.now(), + id: responseId, + model, + transcript: [ + ...transcript, + { + role: 'assistant', + content: getAssistantTranscriptContent(outputText, transcriptToolCalls), + ...(transcriptToolCalls ? { tool_calls: transcriptToolCalls } : {}), + // A client that continues via `previous_response_id` rather than + // replaying `output` never sees the reasoning item, so the session is + // the only place the reasoning can survive into the next turn. + ...(reasoningText ? { reasoning: reasoningText } : {}), + }, + ], + defaults, + upstreamProtocol: 'chat', + }); + + return { + id: responseId, + object: 'response', + created_at: createdAt, + status: 'completed', + model, + output, + output_text: outputText, + usage: mapChatUsageToResponses(upstreamPayload.usage), + metadata: defaults.metadata ?? {}, + previous_response_id: previousResponseId, + }; +}; + +export const buildResponsesWebSearchCallItem = ( + execution: ServerToolExecution | ServerToolInvocation, + status: 'completed' | 'in_progress', + id = `ws_${crypto.randomUUID().replaceAll('-', '')}`, +): Record => ({ + id, + type: 'web_search_call', + status, + action: + execution.type === 'web_search' + ? { type: 'search', query: execution.input.query } + : { + type: 'open_page', + url: + 'result' in execution + ? (execution.result.url ?? execution.input.url) + : execution.input.url, + }, +}); + +/** + * Emits an already-buffered chat payload as a Responses SSE stream. + * + * Used when a request had to be buffered to inspect it — image generation is + * executed locally, so the call cannot be forwarded before it is seen. The + * client still asked for `stream: true`, so the buffered result is replayed as + * the same event sequence a live stream would have produced. + * + * The text is replayed as delta events rather than arriving whole in + * `response.completed`: a client that renders as it reads subscribes to deltas + * and would otherwise show nothing until the turn ends. + */ +export const mapChatResponseToResponsesStream = async ( + upstreamPayload: Record, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + proxyContext: ProxyContext, + imageExecutions: ImageGenerationExecution[], + serverToolExecutions: ServerToolExecution[] = [], +): Promise => { + const payload = await mapChatResponseToResponsesPayload( + proxyContext.accessKeyId, + proxyContext.credentialFilename, + defaults, + transcript, + model, + previousResponseId, + upstreamPayload, + serverToolExecutions, + imageExecutions, + ); + // The mapper creates and persists the session id, so the stream has to reuse + // it: advertising a different one would leave a client unable to continue the + // turn, because nothing was stored under the id it was given. + const responseId = String(payload.id); + const output = payload.output as Array>; + const messageIndex = output.findIndex((item) => item.type === 'message'); + const messageItem = + messageIndex === -1 + ? null + : (output[messageIndex] as { + content?: Array<{ text?: string }>; + id?: string; + }); + const messageText = messageItem?.content?.[0]?.text ?? ''; + const otherItems = output + .map((item, output_index) => ({ item, output_index })) + .filter(({ output_index }) => output_index !== messageIndex); + + // The live path announces a server-tool item as in-progress and narrates its + // lifecycle before closing it, and consumers can subscribe to those events. + // A buffered replay that jumps straight to `done` hides the search entirely + // from a client watching for it. + const serverToolFrames = ({ + item, + output_index, + }: { + item: Record; + output_index: number; + }): Array> => { + const itemId = String(item.id ?? ''); + + if (item.type !== 'web_search_call') { + return [ + { + item, + output_index, + response_id: responseId, + type: 'response.output_item.added', + }, + ]; + } + + return [ + { + item: { ...item, status: 'in_progress' }, + output_index, + response_id: responseId, + type: 'response.output_item.added', + }, + { + item_id: itemId, + output_index, + type: 'response.web_search_call.in_progress', + }, + { + item_id: itemId, + output_index, + type: 'response.web_search_call.searching', + }, + { + item_id: itemId, + output_index, + type: 'response.web_search_call.completed', + }, + ]; + }; + + const frames: Array> = [ + { + response: { ...payload, output: [], status: 'in_progress' }, + type: 'response.created', + }, + { + response: { id: responseId, status: 'in_progress' }, + type: 'response.in_progress', + }, + ...otherItems.flatMap(({ item, output_index }) => + serverToolFrames({ item, output_index }), + ), + ...otherItems.map(({ item, output_index }) => ({ + item, + output_index, + response_id: responseId, + type: 'response.output_item.done', + })), + ]; + + // Mirrors the live path: the message item is announced, filled by deltas, + // then closed. No `content_part` events — the live path does not emit them. + if (messageItem && messageIndex !== -1) { + frames.push({ + item: { ...messageItem, status: 'in_progress' }, + output_index: messageIndex, + response_id: responseId, + type: 'response.output_item.added', + }); + + if (messageText) { + frames.push({ + delta: messageText, + item_id: messageItem.id, + output_index: messageIndex, + response_id: responseId, + type: 'response.output_text.delta', + }); + frames.push({ + item: messageItem, + output_index: messageIndex, + response_id: responseId, + text: messageText, + type: 'response.output_text.done', + }); + } + + frames.push({ + item: messageItem, + output_index: messageIndex, + response_id: responseId, + type: 'response.output_item.done', + }); + } + + frames.push({ + response: { ...payload, id: responseId }, + type: 'response.completed', + }); + + const body = [ + ...frames.map((frame) => eventFrameText(String(frame.type), frame)), + DONE_FRAME_TEXT, + '', + ].join('\n\n'); + + return createSseResponse(body); +}; diff --git a/lib/server/proxy/responses-session.ts b/lib/server/proxy/responses-session.ts new file mode 100644 index 0000000..cefc1ef --- /dev/null +++ b/lib/server/proxy/responses-session.ts @@ -0,0 +1,230 @@ +// --------------------------------------------------------------------------- +// Response session persistence +// --------------------------------------------------------------------------- + +import type { ProxyContext } from './codebuddy'; +import { + deleteStorageJson, + getStorageBackendMeta, + listStorageJson, + readStorageJson, + writeStorageJson, +} from '../storage'; +import type { + ResponseSession, + ResponseSessionMetadata, +} from './responses-types'; + +export const MAX_RESPONSE_SESSIONS = 1_000; +export const RESPONSE_SESSION_TTL_MS = 60 * 60 * 1000; +export const MAX_RESPONSE_SESSION_BYTES = 8 * 1024 * 1024; +export const MAX_RESPONSE_SESSION_TOTAL_BYTES = 64 * 1024 * 1024; +export const MAX_RESPONSE_TRANSCRIPT_MESSAGES = 200; +export const RESPONSE_SESSION_NAMESPACE = 'responses'; +export const RESPONSE_SESSION_INDEX_NAMESPACE = 'response-session-index'; + +const globalResponsesState = globalThis as typeof globalThis & { + __codebuddy2apiResponseSessions__?: Map; + __codebuddy2apiResponseSessionBytes__?: Map; + __codebuddy2apiResponseSessionTotalBytes__?: number; +}; + +export const getSessionStore = (): Map => { + if (!globalResponsesState.__codebuddy2apiResponseSessions__) { + globalResponsesState.__codebuddy2apiResponseSessions__ = new Map(); + } + + return globalResponsesState.__codebuddy2apiResponseSessions__; +}; + +export const getSessionByteStore = (): Map => { + if (!globalResponsesState.__codebuddy2apiResponseSessionBytes__) { + globalResponsesState.__codebuddy2apiResponseSessionBytes__ = new Map(); + } + + return globalResponsesState.__codebuddy2apiResponseSessionBytes__; +}; + +export const getSessionTotalBytes = (): number => { + return globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ ?? 0; +}; + +export const setSessionTotalBytes = (value: number): void => { + globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ = value; +}; + +export const removeLocalResponseSession = (id: string): void => { + const byteStore = getSessionByteStore(); + const store = getSessionStore(); + const bytes = byteStore.get(id) ?? 0; + store.delete(id); + byteStore.delete(id); + setSessionTotalBytes(Math.max(0, getSessionTotalBytes() - bytes)); +}; + +export const pruneResponseSessions = (): void => { + const store = getSessionStore(); + const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; + + for (const [id, session] of store) { + if (session.createdAt <= expiresBefore) { + removeLocalResponseSession(id); + } + } + + while ( + store.size > MAX_RESPONSE_SESSIONS || + getSessionTotalBytes() > MAX_RESPONSE_SESSION_TOTAL_BYTES + ) { + const oldestId = store.keys().next().value; + + // Guard against a byte total that has drifted out of step with the map. + // Without this, an empty map with a positive total makes the removal a + // no-op and spins here forever, blocking the event loop. + if (oldestId === undefined) { + setSessionTotalBytes(0); + break; + } + + removeLocalResponseSession(oldestId); + } +}; + +export const prunePgResponseSessions = async (): Promise => { + const metadataDocuments = await listStorageJson( + RESPONSE_SESSION_INDEX_NAMESPACE, + ); + const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; + const candidates = metadataDocuments + .map((document) => ({ key: document.key, ...document.value })) + .sort((left, right) => left.createdAt - right.createdAt); + const toDelete = candidates.filter( + (candidate) => candidate.createdAt <= expiresBefore, + ); + const remaining = candidates.filter( + (candidate) => candidate.createdAt > expiresBefore, + ); + let totalBytes = remaining.reduce( + (total, candidate) => total + candidate.bytes, + 0, + ); + + while ( + remaining.length > MAX_RESPONSE_SESSIONS || + totalBytes > MAX_RESPONSE_SESSION_TOTAL_BYTES + ) { + const candidate = remaining.shift()!; + toDelete.push(candidate); + totalBytes -= candidate.bytes; + } + + await Promise.all( + toDelete.flatMap((candidate) => [ + deleteStorageJson(RESPONSE_SESSION_NAMESPACE, candidate.key), + deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, candidate.key), + ]), + ); +}; + +export const isPgResponseSessionStore = (): boolean => { + return getStorageBackendMeta().backend === 'pg'; +}; + +export const getResponseSession = async ( + id: string, +): Promise => { + if (isPgResponseSessionStore()) { + const session = await readStorageJson( + RESPONSE_SESSION_NAMESPACE, + id, + ); + if (!session || session.createdAt <= Date.now() - RESPONSE_SESSION_TTL_MS) { + if (session) { + await deleteStorageJson(RESPONSE_SESSION_NAMESPACE, id); + await deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, id); + } + return undefined; + } + return session; + } + + pruneResponseSessions(); + return getSessionStore().get(id); +}; + +export const getValidatedPreviousSession = async ( + previousResponseId: string | null, + accessKeyId: string | null, +): Promise => { + const previousSession = previousResponseId + ? await getResponseSession(previousResponseId) + : undefined; + + if ( + previousResponseId && + (!previousSession || previousSession.accessKeyId !== accessKeyId) + ) { + throw new Error('Unknown or expired previous_response_id'); + } + + return previousSession; +}; + +export const storeResponseSession = async ( + session: ResponseSession, +): Promise => { + const serialized = JSON.stringify(session); + if (Buffer.byteLength(serialized, 'utf8') > MAX_RESPONSE_SESSION_BYTES) { + throw new Error('Response session exceeds the maximum size'); + } + + if (isPgResponseSessionStore()) { + await writeStorageJson(RESPONSE_SESSION_NAMESPACE, session.id, session); + await writeStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, session.id, { + bytes: Buffer.byteLength(serialized, 'utf8'), + createdAt: session.createdAt, + }); + try { + await prunePgResponseSessions(); + } catch (error) { + console.warn('[CodeBuddy2API] Unable to prune Responses sessions', error); + } + return; + } + + const store = getSessionStore(); + const byteStore = getSessionByteStore(); + const previousBytes = byteStore.get(session.id) ?? 0; + const sessionBytes = Buffer.byteLength(serialized, 'utf8'); + store.set(session.id, session); + byteStore.set(session.id, sessionBytes); + setSessionTotalBytes(getSessionTotalBytes() - previousBytes + sessionBytes); + pruneResponseSessions(); +}; + +export const storeUpstreamResponseBinding = async ({ + model, + proxyContext, + responseId, +}: { + model: string; + proxyContext: ProxyContext; + responseId: string; +}): Promise => { + await storeResponseSession({ + accessKeyId: proxyContext.accessKeyId, + credentialFilename: proxyContext.credentialFilename, + createdAt: Date.now(), + defaults: {}, + id: responseId, + model, + transcript: [], + upstreamProtocol: 'responses', + }); +}; + +export const resetResponseSessions = (): void => { + getSessionStore().clear(); + getSessionByteStore().clear(); + setSessionTotalBytes(0); +}; diff --git a/lib/server/proxy/responses-stream.ts b/lib/server/proxy/responses-stream.ts new file mode 100644 index 0000000..0c21f39 --- /dev/null +++ b/lib/server/proxy/responses-stream.ts @@ -0,0 +1,713 @@ +// --------------------------------------------------------------------------- +// Responses SSE stream mapping +// --------------------------------------------------------------------------- + +import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { + createStreamCloser, + readTimeoutFrame, + responsesStreamErrorChunks, + toUpstreamTimeoutMessage, +} from '../shared/upstream-timeout'; +import type { ProxyContext } from './codebuddy'; + +import { storeResponseSession } from './responses-session'; +import { buildResponsesWebSearchCallItem } from './responses-payload'; +import { + createMessageId, + createResponseId, + createResponseOutputId, + createResponseReasoningId, + normalizeToolCallId, +} from './responses-ids'; +import { + buildResponsesToolCallOutputItem, + getResponsesToolCallArgumentDeltaEventType, + hasSupportedLongerToolNamePrefix, +} from './responses-tools'; +import { + buildStreamingAssistantTranscriptToolCalls, + getAssistantTranscriptContent, + getStreamingToolCallCanonicalKey, + getStreamingToolCallLookupKeys, + mapChatUsageToResponses, + REASONING_PREFIX, +} from './responses-transcript'; +import type { + ChatResponseToolCall, + ResponsesServerToolItem, + ResponseSessionDefaults, + StreamingMessageState, + StreamingToolCallState, + TranscriptMessage, +} from './responses-types'; +import { getServerToolExecutions } from './web-search-loop'; +import { MAX_RESPONSE_SESSION_TOTAL_BYTES } from './responses-session'; + +const MAX_STREAM_BUFFER_LENGTH = 1_000_000; +const MAX_STREAM_TEXT_LENGTH = 2_000_000; +const MAX_TOOL_ARGUMENT_LENGTH = 1_000_000; +const MAX_TOOL_NAME_LENGTH = 256; + +export const mapChatStreamToResponsesEventStream = ( + upstreamResponse: Response, + defaults: ResponseSessionDefaults, + transcript: TranscriptMessage[], + model: string, + previousResponseId: string | null, + proxyContext: ProxyContext, + responseId = createResponseId(), + providedServerToolItems?: ResponsesServerToolItem[], + emitOpeningEvents = true, + emitServerToolLifecycle = true, + providedOutputIndexAllocator?: () => number, + rejectErrorPayloads = false, +): Response => { + if (!upstreamResponse.ok || !upstreamResponse.body) { + return upstreamResponse; + } + + const serverToolItems = + providedServerToolItems ?? + getServerToolExecutions(upstreamResponse).map((execution, outputIndex) => { + const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; + + return { + completed: buildResponsesWebSearchCallItem(execution, 'completed', id), + inProgress: buildResponsesWebSearchCallItem( + execution, + 'in_progress', + id, + ), + outputIndex, + }; + }); + let outputText = ''; + // Reasoning accumulated from stream deltas. The delta events alone are not + // replayable — a client needs a reasoning item in the completed output, with + // a blob of its own, to send anything back on the next turn. + let streamedReasoning = ''; + // Claimed on the first reasoning delta, which lands before any text, so the + // item sorts ahead of the message it produced. + let reasoningOutputIndex: number | null = null; + let reasoningItemAdded = false; + // Fixed when the first reasoning delta arrives, so the `output_item.added` + // event and the completed output reference the same id. + let reasoningItemId = ''; + + let nextOutputIndex = + serverToolItems.reduce( + (maximum, item) => Math.max(maximum, item.outputIndex), + -1, + ) + 1; + const allocateOutputIndex = + providedOutputIndexAllocator ?? (() => nextOutputIndex++); + const messageState: StreamingMessageState = { + outputIndex: null, + outputItemId: createMessageId(), + }; + let messageAddedEmitted = false; + const toolCallStates = new Map(); + const toolCallStateKeys = new Map(); + let latestUsage: unknown = null; + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + const closer = createStreamCloser(); + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + + const stream = new ReadableStream({ + start: (controller) => { + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + let totalToolArgumentLength = 0; + let streamRejected = false; + + const enqueueEvent = (payload: Record): void => { + const eventType = + typeof payload.type === 'string' ? payload.type : 'message'; + controller.enqueue( + encoder.encode( + `event: ${eventType}\ndata: ${JSON.stringify(payload)}\n\n`, + ), + ); + }; + + const buildStreamingMessageItem = ( + status: 'completed' | 'in_progress', + ): Record => ({ + id: messageState.outputItemId, + type: 'message', + role: 'assistant', + status, + content: [ + { + type: 'output_text', + text: outputText, + annotations: [], + }, + ], + }); + + // Carries the reasoning verbatim rather than encrypted — same reasoning as + // the non-streaming item above. + const buildStreamingReasoningItem = (): Record => ({ + id: reasoningItemId, + type: 'reasoning', + summary: [{ type: 'summary_text', text: streamedReasoning }], + encrypted_content: `${REASONING_PREFIX}${streamedReasoning}`, + status: 'completed', + }); + + const ensureReasoningItemAdded = (): void => { + if (reasoningItemAdded) { + return; + } + + reasoningOutputIndex ??= allocateOutputIndex(); + enqueueEvent({ + type: 'response.output_item.added', + item: buildStreamingReasoningItem(), + output_index: reasoningOutputIndex, + response_id: responseId, + }); + reasoningItemAdded = true; + }; + + const ensureMessageAdded = (): void => { + if (messageAddedEmitted) { + return; + } + + messageState.outputIndex ??= allocateOutputIndex(); + enqueueEvent({ + type: 'response.output_item.added', + item: buildStreamingMessageItem('in_progress'), + output_index: messageState.outputIndex, + response_id: responseId, + }); + messageAddedEmitted = true; + }; + + if (emitOpeningEvents) { + enqueueEvent({ + type: 'response.created', + response: { + id: responseId, + object: 'response', + created_at: Math.floor(Date.now() / 1000), + model, + output: [], + }, + }); + enqueueEvent({ + type: 'response.in_progress', + response: { + id: responseId, + status: 'in_progress', + }, + }); + } + if (emitServerToolLifecycle) { + serverToolItems.forEach(({ completed, inProgress, outputIndex }) => { + const itemId = String(inProgress.id); + enqueueEvent({ + type: 'response.output_item.added', + item: inProgress, + output_index: outputIndex, + response_id: responseId, + }); + enqueueEvent({ + type: 'response.web_search_call.in_progress', + item_id: itemId, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.web_search_call.searching', + item_id: itemId, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.web_search_call.completed', + item_id: itemId, + output_index: outputIndex, + }); + enqueueEvent({ + type: 'response.output_item.done', + item: completed, + output_index: outputIndex, + response_id: responseId, + }); + }); + } + + const maybeEmitToolCallAdded = ( + toolCallState: StreamingToolCallState, + allowIncompleteName = false, + ): void => { + if (toolCallState.addedEmitted) { + return; + } + + const shouldWaitForInitialName = + !allowIncompleteName && + Boolean(defaults.tools?.length) && + toolCallState.name.length === 0; + const shouldWaitForMoreName = + !allowIncompleteName && + defaults.tools?.length && + toolCallState.name.length > 0 && + hasSupportedLongerToolNamePrefix(defaults.tools, toolCallState.name); + + if (shouldWaitForInitialName || shouldWaitForMoreName) { + return; + } + + enqueueEvent({ + type: 'response.output_item.added', + item: buildResponsesToolCallOutputItem(defaults.tools, { + arguments: '', + callId: toolCallState.callId, + id: toolCallState.outputItemId, + name: toolCallState.name || 'function', + status: 'in_progress', + }), + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + + toolCallState.addedEmitted = true; + toolCallState.pendingArgumentDeltas.forEach((delta) => { + enqueueEvent({ + type: getResponsesToolCallArgumentDeltaEventType( + defaults.tools, + toolCallState.name, + ), + delta, + item_id: toolCallState.outputItemId, + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + }); + toolCallState.pendingArgumentDeltas = []; + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + const transcriptToolCalls = + buildStreamingAssistantTranscriptToolCalls( + [...toolCallStates.values()], + defaults.tools, + ); + try { + await storeResponseSession({ + accessKeyId: proxyContext.accessKeyId, + credentialFilename: proxyContext.credentialFilename, + createdAt: Date.now(), + id: responseId, + model, + transcript: [ + ...transcript, + { + role: 'assistant', + content: getAssistantTranscriptContent( + outputText, + transcriptToolCalls, + ), + ...(transcriptToolCalls + ? { tool_calls: transcriptToolCalls } + : {}), + // Same reason as the non-streaming path: a client that + // continues via `previous_response_id` instead of + // replaying `output` would otherwise lose the reasoning. + ...(streamedReasoning + ? { reasoning: streamedReasoning } + : {}), + }, + ], + defaults, + upstreamProtocol: 'chat', + }); + } catch (error) { + console.error( + '[CodeBuddy2API] Failed to persist Responses session', + error, + ); + enqueueEvent({ + type: 'response.error', + error: { message: 'Failed to persist response session' }, + }); + controller.enqueue(encodeDoneFrame()); + releaseReader(); + controller.close(); + return; + } + [...toolCallStates.values()].forEach((toolCallState) => { + maybeEmitToolCallAdded(toolCallState, true); + enqueueEvent({ + type: 'response.output_item.done', + item: buildResponsesToolCallOutputItem(defaults.tools, { + arguments: toolCallState.arguments, + callId: toolCallState.callId, + id: toolCallState.outputItemId, + name: toolCallState.name || 'function', + status: 'completed', + }), + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + enqueueEvent({ + type: getResponsesToolCallArgumentDeltaEventType( + defaults.tools, + toolCallState.name, + ).replace('.delta', '.done'), + arguments: toolCallState.arguments, + item_id: toolCallState.outputItemId, + output_index: toolCallState.outputIndex, + response_id: responseId, + }); + }); + if (outputText) { + ensureMessageAdded(); + enqueueEvent({ + type: 'response.output_text.done', + item: buildStreamingMessageItem('completed'), + output_index: messageState.outputIndex, + response_id: responseId, + text: outputText, + }); + enqueueEvent({ + type: 'response.output_item.done', + item: buildStreamingMessageItem('completed'), + output_index: messageState.outputIndex, + response_id: responseId, + }); + } + enqueueEvent({ + type: 'response.completed', + response: { + id: responseId, + status: 'completed', + output_text: outputText, + previous_response_id: previousResponseId, + usage: mapChatUsageToResponses(latestUsage), + output: [ + ...serverToolItems.map(({ completed, outputIndex }) => ({ + item: completed, + outputIndex, + })), + // Streamed reasoning needs the same replayable item the + // non-streaming path emits. It sorts ahead of the message by + // taking the next index before the message claims its own — + // the deltas come first on the wire, so the item order has + // to match or a client replaying `output` scrambles it. + ...(streamedReasoning && reasoningOutputIndex !== null + ? [ + { + item: buildStreamingReasoningItem(), + outputIndex: reasoningOutputIndex, + }, + ] + : []), + ...(outputText && messageState.outputIndex !== null + ? [ + { + item: buildStreamingMessageItem('completed'), + outputIndex: messageState.outputIndex, + }, + ] + : []), + ...[...toolCallStates.values()].map((toolCallState) => ({ + item: buildResponsesToolCallOutputItem(defaults.tools, { + arguments: toolCallState.arguments, + callId: toolCallState.callId, + id: toolCallState.outputItemId, + name: toolCallState.name || 'function', + status: 'completed', + }), + outputIndex: toolCallState.outputIndex, + })), + ] + .sort((left, right) => left.outputIndex - right.outputIndex) + .map(({ item }) => item), + }, + }); + controller.enqueue(encodeDoneFrame()); + releaseReader(); + controller.close(); + return; + } + + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_BUFFER_LENGTH) { + buffer = ''; + } + + for (const frame of frames) { + if (streamRejected) { + break; + } + if (frame.length > MAX_STREAM_BUFFER_LENGTH) { + continue; + } + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + continue; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + // The upstream here is the chat pipeline, which reports a deadline + // as a terminal error chunk and closes cleanly. Surfacing it keeps + // the client from seeing an empty successful response. + const upstreamError = readTimeoutFrame(frame); + + if (upstreamError !== null) { + streamRejected = true; + enqueueEvent({ + type: 'response.error', + error: { message: upstreamError }, + }); + break; + } + + try { + const payload = JSON.parse(raw) as { + choices?: Array<{ + delta?: { + content?: string; + reasoning_content?: string; + tool_calls?: ChatResponseToolCall[]; + }; + }>; + error?: unknown; + usage?: unknown; + }; + if (rejectErrorPayloads && payload.error) { + const error = + typeof payload.error === 'object' + ? (payload.error as { message?: unknown }) + : null; + streamRejected = true; + enqueueEvent({ + type: 'response.error', + error: { + message: + typeof error?.message === 'string' + ? error.message + : typeof payload.error === 'string' + ? payload.error + : 'Upstream request failed', + }, + }); + break; + } + // The final upstream chunk carries the aggregated usage, so + // remember it for the downstream response.completed event. + if (payload.usage !== undefined) { + latestUsage = payload.usage; + } + const delta = payload.choices?.[0]?.delta; + + if (delta?.content) { + ensureMessageAdded(); + outputText = `${outputText}${delta.content}`; + if (outputText.length > MAX_STREAM_TEXT_LENGTH) { + throw new Error('Response output exceeds the maximum size'); + } + enqueueEvent({ + type: 'response.output_text.delta', + delta: delta.content, + // Send only the item reference: embedding the accumulated + // text re-serializes it on every delta, which makes the + // enqueued volume quadratic in the output size. The full + // text still arrives intact in response.output_text.done + // and response.completed. + item_id: messageState.outputItemId, + output_index: messageState.outputIndex, + response_id: responseId, + }); + } + + if (delta?.reasoning_content) { + reasoningItemId ||= createResponseReasoningId(); + streamedReasoning += delta.reasoning_content; + ensureReasoningItemAdded(); + enqueueEvent({ + type: 'response.reasoning_text.delta', + delta: delta.reasoning_content, + response_id: responseId, + }); + } + + delta?.tool_calls?.forEach((toolCall, position) => { + const lookupKeys = getStreamingToolCallLookupKeys( + toolCall, + position, + ); + const existingCanonicalKey = lookupKeys + .map((key) => toolCallStateKeys.get(key) ?? key) + .find((key) => toolCallStates.has(key)); + const canonicalKey = + existingCanonicalKey ?? + getStreamingToolCallCanonicalKey(toolCall, position); + const existing = toolCallStates.get(canonicalKey); + const outputIndex = existing + ? existing.outputIndex + : allocateOutputIndex(); + const current = existing ?? { + addedEmitted: false, + arguments: '', + canonicalKey, + callId: normalizeToolCallId(toolCall.id, outputIndex), + name: '', + outputIndex, + outputItemId: createResponseOutputId(), + pendingArgumentDeltas: [], + }; + + if (toolCall.function?.name) { + if ( + current.name.length + toolCall.function.name.length > + MAX_TOOL_NAME_LENGTH + ) { + throw new Error( + 'Response tool name exceeds the maximum size', + ); + } + current.name += toolCall.function.name; + } + maybeEmitToolCallAdded(current); + + if (toolCall.function?.arguments) { + if ( + current.arguments.length + + toolCall.function.arguments.length > + MAX_TOOL_ARGUMENT_LENGTH + ) { + throw new Error( + 'Response tool arguments exceed the maximum size', + ); + } + if ( + totalToolArgumentLength + + toolCall.function.arguments.length > + MAX_RESPONSE_SESSION_TOTAL_BYTES + ) { + throw new Error( + 'Response tool arguments exceed the maximum size', + ); + } + totalToolArgumentLength += toolCall.function.arguments.length; + current.arguments = `${current.arguments}${toolCall.function.arguments}`; + if (current.addedEmitted) { + enqueueEvent({ + type: getResponsesToolCallArgumentDeltaEventType( + defaults.tools, + current.name, + ), + delta: toolCall.function.arguments, + item_id: current.outputItemId, + output_index: current.outputIndex, + response_id: responseId, + }); + } else { + current.pendingArgumentDeltas.push( + toolCall.function.arguments, + ); + } + } + + toolCallStates.set(canonicalKey, current); + lookupKeys.forEach((key) => { + toolCallStateKeys.set(key, current.canonicalKey); + }); + }); + } catch (error) { + if ( + error instanceof Error && + error.message.includes('maximum size') + ) { + streamRejected = true; + } + console.error( + '[CodeBuddy2API] Failed to parse upstream SSE frame', + { + route: '/v1/responses', + frame: raw.slice(0, 1000), + }, + ); + enqueueEvent({ + type: 'response.error', + error: { + message: 'Failed to parse upstream SSE frame', + }, + }); + } + } + + if (streamRejected) { + try { + await reader!.cancel(); + } finally { + releaseReader(); + controller.close(); + } + return; + } + } + }; + + void pump().catch((error) => { + if (cancelled) return; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage === null) { + closer.mark(); + controller.error(error); + return; + } + + streamRejected = true; + void reader?.cancel().then( + () => undefined, + () => undefined, + ); + releaseReader(); + closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + releaseReader(); + } + }, + }); + + return createSseResponse(stream, { status: 200 }); +}; diff --git a/lib/server/proxy/responses-tools.ts b/lib/server/proxy/responses-tools.ts new file mode 100644 index 0000000..903a1a7 --- /dev/null +++ b/lib/server/proxy/responses-tools.ts @@ -0,0 +1,582 @@ +// --------------------------------------------------------------------------- +// Responses <-> Chat tool translation +// --------------------------------------------------------------------------- + +import { createErrorResponse } from '../shared/http'; +import { + buildWebFetchToolDefinition, + buildWebSearchToolDefinition, + markServerTool, + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_FETCH_TOOL_TYPE_PREFIX, + WEB_SEARCH_TOOL_NAME, + WEB_SEARCH_TOOL_TYPE_PREFIX, +} from '../search/tool'; +import { + buildImageGenerationChatTool, + IMAGE_GENERATION_CHAT_TOOL_NAME, + IMAGE_GENERATION_TOOL_TYPE, +} from './image-generation'; +import type { + ResponsesRequestBody, + SupportedChatTool, + SupportedResponsesTool, + TranscriptMessage, +} from './responses-types'; + +export const TOOL_SEARCH_PROXY_NAME = 'tool_search'; +export const CUSTOM_TOOL_INPUT_FIELD = 'input'; +export const CUSTOM_TOOL_INPUT_DESCRIPTION = + 'Raw string input for the original custom tool.'; + +export const flattenNamespaceToolName = ( + namespace: string, + name: string, +): string => { + return `${namespace}__${name}`; +}; + +export const extractFunctionDefinition = ( + tool: Record, +): Record | null => { + const nested = + typeof tool.function === 'object' && tool.function !== null + ? (tool.function as Record) + : {}; + + const name = nested.name ?? tool.name; + if (typeof name !== 'string' || name.length === 0) { + return null; + } + + const functionDef: Record = { name }; + + const description = nested.description ?? tool.description; + if (description !== undefined) { + functionDef.description = description; + } + + const parameters = nested.parameters ?? tool.parameters; + if (parameters !== undefined) { + functionDef.parameters = parameters; + } + + const strict = nested.strict ?? tool.strict; + if (strict !== undefined) { + functionDef.strict = strict; + } + + return functionDef; +}; + +export const buildCustomToolDefinition = ( + tool: Record, +): Record | null => { + const name = typeof tool.name === 'string' ? tool.name.trim() : ''; + + if (!name) { + return null; + } + + const description = + typeof tool.description === 'string' && tool.description.trim() + ? tool.description + : `Custom tool ${name}`; + + return { + name, + description, + parameters: { + type: 'object', + properties: { + [CUSTOM_TOOL_INPUT_FIELD]: { + type: 'string', + description: CUSTOM_TOOL_INPUT_DESCRIPTION, + }, + }, + required: [CUSTOM_TOOL_INPUT_FIELD], + }, + }; +}; + +export const buildToolSearchDefinition = (): Record => { + return { + name: TOOL_SEARCH_PROXY_NAME, + description: + 'Search and load Codex tools, plugins, connectors, and MCP namespaces for the current task.', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'Search query for tools or connectors to load.', + }, + limit: { + type: 'integer', + description: 'Maximum number of tool groups to return.', + }, + }, + required: ['query'], + }, + }; +}; + +export const toSupportedChatTool = ( + tool: SupportedResponsesTool, + namespace?: string, +): SupportedChatTool[] => { + const toolType = typeof tool.type === 'string' ? tool.type : 'function'; + + // Server-side search and fetch carry no function schema, so the generic + // branch below drops them. Emit them as functions unconditionally and let + // the proxy loop resolve the configured backend asynchronously. SearXNG + // needs local configuration, while CodeBuddy search does not. + if ( + normalizeToolName(toolType).startsWith( + normalizeToolName(WEB_SEARCH_TOOL_TYPE_PREFIX), + ) + ) { + const definition = buildWebSearchToolDefinition(); + + return [ + { + chatName: WEB_SEARCH_TOOL_NAME, + kind: 'function', + originalName: WEB_SEARCH_TOOL_NAME, + serverDeclared: true, + tool: definition, + }, + ]; + } + + // Fetch needs no deployment-level configuration — the local backend is always + // available and the CodeBuddy backend needs only a credential — so it is + // advertised unconditionally and gated later by the enable toggle. + if ( + normalizeToolName(toolType).startsWith( + normalizeToolName(WEB_FETCH_TOOL_TYPE_PREFIX), + ) + ) { + const definition = buildWebFetchToolDefinition(); + + return [ + { + chatName: WEB_FETCH_TOOL_NAME, + kind: 'function', + originalName: WEB_FETCH_TOOL_NAME, + serverDeclared: true, + tool: definition, + }, + ]; + } + + // Image generation has no chat-protocol equivalent, so the declaration is + // rewritten as a function and the call is executed by the proxy. It is + // advertised only on the chat path: the responses passthrough hands the + // native declaration straight to the upstream, which supports it. + if (toolType === IMAGE_GENERATION_TOOL_TYPE) { + return [ + { + chatName: IMAGE_GENERATION_CHAT_TOOL_NAME, + kind: 'function' as const, + originalName: IMAGE_GENERATION_TOOL_TYPE, + serverDeclared: true, + tool: buildImageGenerationChatTool(), + }, + ]; + } + + if (toolType === 'namespace') { + const namespaceName = typeof tool.name === 'string' ? tool.name.trim() : ''; + const children = ( + Array.isArray(tool.tools) + ? tool.tools + : Array.isArray(tool.children) + ? tool.children + : [] + ).filter((item): item is SupportedResponsesTool => { + return Boolean(item && typeof item === 'object'); + }); + + if (!namespaceName || !children.length) { + return []; + } + + return children.flatMap((child) => + toSupportedChatTool(child, namespaceName), + ); + } + + if (toolType === 'tool_search') { + const definition = buildToolSearchDefinition(); + return [ + { + chatName: TOOL_SEARCH_PROXY_NAME, + kind: 'tool_search', + originalName: TOOL_SEARCH_PROXY_NAME, + tool: definition, + }, + ]; + } + + if (toolType === 'custom') { + const definition = buildCustomToolDefinition(tool); + + if (!definition || typeof definition.name !== 'string') { + return []; + } + + return [ + { + chatName: definition.name, + kind: 'custom', + originalName: definition.name, + tool: definition, + }, + ]; + } + + const functionDef = extractFunctionDefinition(tool); + + if (!functionDef || typeof functionDef.name !== 'string') { + return []; + } + + const originalName = functionDef.name; + const chatName = namespace + ? flattenNamespaceToolName(namespace, originalName) + : toolType === 'mcp' && + typeof tool.server_label === 'string' && + tool.server_label.trim() + ? flattenNamespaceToolName(tool.server_label.trim(), originalName) + : originalName; + + return [ + { + chatName, + kind: toolType === 'mcp' ? 'mcp' : 'function', + namespace: + namespace || + (typeof tool.server_label === 'string' ? tool.server_label : undefined), + originalName, + serverLabel: + toolType === 'mcp' && typeof tool.server_label === 'string' + ? tool.server_label + : undefined, + tool: { + ...functionDef, + name: chatName, + }, + }, + ]; +}; + +/** + * True when the client declared an `image_generation` tool. Gating on this + * keeps the ordinary path free of an extra upstream round trip. + */ +export const hasImageGenerationTool = ( + tools: ResponsesRequestBody['tools'], +): boolean => { + return Boolean( + tools?.some( + (tool) => + typeof tool?.type === 'string' && + tool.type.toLowerCase().replaceAll('-', '_') === + IMAGE_GENERATION_TOOL_TYPE, + ), + ); +}; + +export const getSupportedChatTools = ( + tools: ResponsesRequestBody['tools'], +): SupportedChatTool[] => { + if (!tools?.length) { + return []; + } + + return tools.flatMap((tool) => toSupportedChatTool(tool)); +}; + +export const findSupportedToolByName = ( + tools: ResponsesRequestBody['tools'], + name: string, +): SupportedChatTool | null => { + if (!tools?.length || !name) { + return null; + } + + return ( + getSupportedChatTools(tools).find( + (tool) => tool.chatName === name || tool.originalName === name, + ) ?? null + ); +}; + +export const hasSupportedLongerToolNamePrefix = ( + tools: ResponsesRequestBody['tools'], + prefix: string, +): boolean => { + if (!tools?.length || !prefix) { + return false; + } + + return getSupportedChatTools(tools).some((tool) => { + const name = tool.chatName; + return ( + typeof name === 'string' && + name.length > prefix.length && + name.startsWith(prefix) + ); + }); +}; + +export const buildResponsesToolCallOutputItem = ( + tools: ResponsesRequestBody['tools'], + toolCall: { + arguments: string; + callId: string; + id: string; + name: string; + status: 'completed' | 'in_progress'; + }, +): Record => { + const originalTool = findSupportedToolByName(tools, toolCall.name); + const itemType = originalTool?.kind === 'mcp' ? 'mcp_call' : 'function_call'; + const item: Record = { + id: toolCall.id, + type: itemType, + call_id: toolCall.callId, + name: originalTool?.originalName ?? toolCall.name ?? 'function', + arguments: toolCall.arguments, + status: toolCall.status, + }; + + if (originalTool?.kind === 'mcp' && originalTool.serverLabel) { + item.server_label = originalTool.serverLabel; + } + + if (originalTool?.kind === 'function' && originalTool.namespace) { + item.namespace = originalTool.namespace; + } + + return item; +}; + +export const getResponsesToolCallArgumentDeltaEventType = ( + tools: ResponsesRequestBody['tools'], + name: string, +): + | 'response.function_call_arguments.delta' + | 'response.mcp_call_arguments.delta' => { + return findSupportedToolByName(tools, name)?.kind === 'mcp' + ? 'response.mcp_call_arguments.delta' + : 'response.function_call_arguments.delta'; +}; + +export const translateResponsesToolsToChat = ( + tools: ResponsesRequestBody['tools'], +): unknown[] | undefined => { + if (!tools?.length) { + return undefined; + } + + const supported = getSupportedChatTools(tools); + if (!supported.length) { + return undefined; + } + + return supported.map((tool) => { + return { + type: 'function', + function: tool.tool, + ...(tool.serverDeclared ? markServerTool({}) : {}), + }; + }); +}; + +export const translateResponsesToolChoiceToChat = ( + toolChoice: unknown, +): unknown => { + if (typeof toolChoice !== 'object' || toolChoice === null) { + return toolChoice; + } + + const choice = toolChoice as Record; + + if ( + choice.type === 'function' && + choice.function && + typeof choice.function === 'object' + ) { + return toolChoice; + } + + if ( + (choice.type === 'auto' || + choice.type === 'none' || + choice.type === 'required') && + typeof choice.type === 'string' + ) { + return choice.type; + } + + // Responses API selects a function by name: + // {type: 'function', name: 'fn'} -> chat schema {type: 'function', function: {name: 'fn'}} + if (typeof choice.name === 'string') { + return { + type: 'function', + function: { name: choice.name }, + }; + } + + return toolChoice; +}; + +export const translateResponsesToolChoiceToChatWithTools = ( + tools: ResponsesRequestBody['tools'], + toolChoice: unknown, +): unknown => { + const translated = translateResponsesToolChoiceToChat(toolChoice); + + if (typeof translated !== 'object' || translated === null) { + return translated; + } + + const choice = translated as Record; + + if ( + choice.type === 'function' && + typeof choice.function === 'object' && + choice.function !== null + ) { + const functionChoice = choice.function as Record; + if (typeof functionChoice.name === 'string') { + return { + ...choice, + function: { + ...functionChoice, + name: resolveChatToolName(tools, functionChoice.name), + }, + }; + } + } + + return translated; +}; + +export const getNamedToolChoice = (toolChoice: unknown): string | null => { + if (typeof toolChoice !== 'object' || toolChoice === null) { + return null; + } + + const choice = toolChoice as Record; + + if (typeof choice.name === 'string' && choice.name.length > 0) { + return choice.name; + } + + if ( + choice.type === 'function' && + typeof choice.function === 'object' && + choice.function !== null && + typeof (choice.function as Record).name === 'string' + ) { + return (choice.function as Record).name; + } + + return null; +}; + +export const getResponsesCompatibilityError = ( + tools: ResponsesRequestBody['tools'], + toolChoice: unknown, +): Response | null => { + const supportedTools = getSupportedChatTools(tools); + + if (toolChoice === 'required' && supportedTools.length === 0) { + return createErrorResponse( + 400, + 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', + ); + } + + if (typeof toolChoice === 'object' && toolChoice !== null) { + const choice = toolChoice as Record; + const isPretranslatedFunctionChoice = + choice.type === 'function' && + typeof choice.function === 'object' && + choice.function !== null; + const isSimpleChoiceType = + choice.type === 'auto' || + choice.type === 'none' || + choice.type === 'required'; + const isNamedFunctionLikeChoice = typeof choice.name === 'string'; + + if ( + !isPretranslatedFunctionChoice && + !isSimpleChoiceType && + !isNamedFunctionLikeChoice + ) { + return createErrorResponse( + 400, + 'Unsupported Responses tool_choice for this /v1/responses adapter', + ); + } + + if (choice.type === 'required' && supportedTools.length === 0) { + return createErrorResponse( + 400, + 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', + ); + } + } + + const namedToolChoice = getNamedToolChoice(toolChoice); + if (namedToolChoice) { + const supportedNames = new Set( + supportedTools + .map((tool) => tool.originalName) + .filter((name): name is string => typeof name === 'string'), + ); + + if (!supportedNames.has(namedToolChoice)) { + return createErrorResponse( + 400, + 'tool_choice references a tool that is not available to this /v1/responses adapter', + ); + } + } + + return null; +}; + +export const resolveChatToolName = ( + tools: ResponsesRequestBody['tools'], + name: string, +): string => { + return findSupportedToolByName(tools, name)?.chatName ?? name; +}; + +export const normalizeTranscriptMessageToolNames = ( + transcript: TranscriptMessage[], + tools: ResponsesRequestBody['tools'], +): TranscriptMessage[] => { + return transcript.map((message) => { + if (!message.tool_calls?.length) { + return message; + } + + return { + ...message, + tool_calls: message.tool_calls.map((toolCall) => ({ + ...toolCall, + function: { + ...toolCall.function, + name: resolveChatToolName(tools, toolCall.function.name), + }, + })), + }; + }); +}; diff --git a/lib/server/proxy/responses-transcript.ts b/lib/server/proxy/responses-transcript.ts new file mode 100644 index 0000000..9af7c2c --- /dev/null +++ b/lib/server/proxy/responses-transcript.ts @@ -0,0 +1,459 @@ +// --------------------------------------------------------------------------- +// Transcript construction and usage mapping +// --------------------------------------------------------------------------- + +import { getDefaultModel } from '../domain/config'; +import { stringifyContent } from '../shared/content'; +import { extractImageUrl, isImageContentPart } from './codebuddy'; +import { createResponseOutputId, normalizeToolCallId } from './responses-ids'; +import { + getValidatedPreviousSession, + MAX_RESPONSE_TRANSCRIPT_MESSAGES, +} from './responses-session'; +import { findSupportedToolByName } from './responses-tools'; +import type { + ChatContentPart, + ChatResponseToolCall, + ResponsesInputItem, + ResponsesRequestBody, + ResponseSession, + ResponseSessionDefaults, + StreamingToolCallState, + TranscriptContent, + TranscriptMessage, +} from './responses-types'; + +export const buildAssistantTranscriptToolCalls = ( + toolCalls: ChatResponseToolCall[], + tools?: ResponsesRequestBody['tools'], +): TranscriptMessage['tool_calls'] | undefined => { + if (!toolCalls.length) { + return undefined; + } + + return toolCalls.map((toolCall, index) => ({ + id: normalizeToolCallId(toolCall.id, index), + type: 'function', + function: { + name: + findSupportedToolByName(tools, toolCall.function?.name ?? '') + ?.originalName ?? + toolCall.function?.name ?? + 'function', + arguments: toolCall.function?.arguments ?? '', + }, + })); +}; + +export const buildStreamingAssistantTranscriptToolCalls = ( + toolCallStates: StreamingToolCallState[], + tools?: ResponsesRequestBody['tools'], +): TranscriptMessage['tool_calls'] | undefined => { + if (!toolCallStates.length) { + return undefined; + } + + return toolCallStates.map((toolCallState) => ({ + id: toolCallState.callId, + type: 'function', + function: { + arguments: toolCallState.arguments, + name: + findSupportedToolByName(tools, toolCallState.name)?.originalName ?? + toolCallState.name, + }, + })); +}; + +export const getAssistantTranscriptContent = ( + outputText: string, + toolCalls: TranscriptMessage['tool_calls'] | undefined, +): string | null => { + return toolCalls?.length ? outputText || null : outputText; +}; + +/** + * Keeps image parts as structured content so the chat path can rebuild them + * upstream. Text parts are still flattened: the transcript is persisted across + * turns and replayed as Chat messages, and the Responses converter only + * recognises images in the OpenAI `image_url` shape. + */ +export const mapInputContentToTranscriptContent = ( + content: unknown, +): TranscriptContent | null => { + if (typeof content === 'string') { + return content; + } + + if (!Array.isArray(content)) { + return null; + } + + const parts = content.filter((part) => part !== null && part !== undefined); + + if (!parts.some(isImageContentPart)) { + return null; + } + + const mapped = parts.flatMap((part): ChatContentPart[] => { + if (typeof part === 'string') { + return [part]; + } + + if (isImageContentPart(part)) { + const imageUrl = extractImageUrl(part); + + return imageUrl + ? [{ image_url: { url: imageUrl }, type: 'image_url' }] + : []; + } + + if (part && typeof part === 'object' && 'text' in part) { + return [String((part as { text?: unknown }).text ?? '')]; + } + + return []; + }); + + return mapped.length ? mapped : null; +}; + +/** + * Marks an `encrypted_content` value we minted, so we can tell it apart from a + * blob issued by someone else. + * + * Not a security measure. Codex never opens this field — it only echoes it — so + * plaintext round-trips fine, but a marker is what stops us from reading a + * genuinely encrypted blob as if it were reasoning text. + */ +export const REASONING_PREFIX = 'cbreason1:'; + +/** + * Pulls readable reasoning out of a replayed `reasoning` item. + * + * Only values we minted are used: anything else — an OpenAI-issued blob, say — + * is opaque ciphertext, and forwarding it upstream would send gibberish where + * reasoning belongs. The summary is the fallback in that case. + * + * The Agents SDK sends summaries as `summary: [{type: 'summary_text', text}]`, + * so a client that never received our blob still gets its reasoning through. + */ +export const extractReasoningFromItem = (item: ResponsesInputItem): string => { + const blob = item.encrypted_content; + + if (typeof blob === 'string' && blob.startsWith(REASONING_PREFIX)) { + return blob.slice(REASONING_PREFIX.length); + } + + if (!Array.isArray(item.summary)) { + return ''; + } + + return item.summary + .map((entry) => { + if (typeof entry === 'string') { + return entry; + } + + if (entry && typeof entry === 'object' && 'text' in entry) { + return String((entry as { text?: unknown }).text ?? ''); + } + + return ''; + }) + .join(''); +}; + +export const mapInputItemToMessage = ( + item: ResponsesInputItem, +): TranscriptMessage | null => { + if (item.type === 'reasoning' || item.type === 'compaction') { + // Reasoning is not a message. Without this branch the item fell through to + // the plain-message case at the bottom, where it has neither `role` nor + // `content` — becoming an empty `{role:'user', content:''}` entry that the + // chat upstream sees as a turn the user never sent, repeated on every + // later turn of the conversation. + // + // Signal the reasoning back to the caller instead, which attaches it to the + // assistant message it accompanies. Returning `null` when there is nothing + // to recover keeps an empty reasoning item from emitting a message at all. + const reasoning = extractReasoningFromItem(item); + + return reasoning ? { role: 'assistant', content: null, reasoning } : null; + } + + if (item.type === 'function_call' || item.type === 'mcp_call') { + return { + role: 'assistant', + content: null, + tool_calls: [ + { + id: item.call_id ?? createResponseOutputId(), + type: 'function', + function: { + name: item.name ?? 'function', + arguments: item.arguments ?? '', + }, + }, + ], + }; + } + + if (item.type === 'function_call_output' || item.type === 'mcp_call_output') { + // A tool may return an image, e.g. a screenshot. Keep it structured so the + // Responses converter can rebuild it as an image; stringifying would hand + // the model the base64 payload as text. + const outputContent = + mapInputContentToTranscriptContent(item.output) ?? + stringifyContent(item.output); + + if (item.call_id) { + return { + role: 'tool', + content: outputContent, + tool_call_id: item.call_id, + }; + } + + return { + role: 'user', + content: outputContent, + }; + } + + if (item.type === 'mcp_approval_response') { + return { + role: 'user', + content: JSON.stringify(item), + }; + } + + // Every other item type returns above, so what is left is a plain message: + // either one with a declared `type: 'message'`, or one carrying only + // `role`/`content`. Images are kept structured so the chat path can rebuild + // them; a message without an image stays flattened. + const imageContent = mapInputContentToTranscriptContent(item.content); + + if (imageContent !== null) { + return { + role: item.role ?? 'user', + content: imageContent, + }; + } + + return { + role: item.role ?? 'user', + content: item.text ?? stringifyContent(item.content), + }; +}; + +export const getStreamingToolCallCanonicalKey = ( + toolCall: ChatResponseToolCall, + position: number, +): string => { + if (toolCall.id) { + return `id:${toolCall.id}`; + } + + if (typeof toolCall.index === 'number') { + return `index:${toolCall.index}`; + } + + return `position:${position}`; +}; + +export const getStreamingToolCallLookupKeys = ( + toolCall: ChatResponseToolCall, + position: number, +): string[] => { + if (toolCall.id || typeof toolCall.index === 'number') { + return [ + toolCall.id ? `id:${toolCall.id}` : null, + typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, + ].filter((key): key is string => key !== null); + } + + return [`position:${position}`]; +}; + +export const prepareTranscript = async ( + body: ResponsesRequestBody, + accessKeyId: string | null, + previousSession?: ResponseSession, +): Promise<{ + defaults: ResponseSessionDefaults; + model: string; + transcript: TranscriptMessage[]; + previousResponseId: string | null; +}> => { + const previousResponseId = body.previous_response_id ?? null; + const resolvedPreviousSession = + previousSession ?? + (await getValidatedPreviousSession(previousResponseId, accessKeyId)); + + const transcript = (resolvedPreviousSession?.transcript ?? []).slice( + -MAX_RESPONSE_TRANSCRIPT_MESSAGES, + ); + // Reasoning recovered from replayed reasoning items, awaiting the assistant + // message it belongs to. Declared here so it spans the whole input array. + let pendingReasoning = ''; + while (transcript[0]?.role === 'tool') { + transcript.shift(); + } + const model = + typeof body.model === 'string' && body.model.trim() + ? body.model + : (resolvedPreviousSession?.model ?? (await getDefaultModel())); + const additionalTools = Array.isArray(body.input) + ? body.input.flatMap((item) => + item?.type === 'additional_tools' && Array.isArray(item.tools) + ? item.tools + : [], + ) + : []; + const baseTools = body.tools ?? resolvedPreviousSession?.defaults.tools; + const requestTools = [...(baseTools ?? []), ...additionalTools]; + const defaults = { + instructions: + body.instructions ?? + resolvedPreviousSession?.defaults.instructions ?? + undefined, + metadata: + body.metadata ?? resolvedPreviousSession?.defaults.metadata ?? undefined, + tools: requestTools.length > 0 ? requestTools : baseTools, + tool_choice: + body.tool_choice ?? + resolvedPreviousSession?.defaults.tool_choice ?? + undefined, + }; + + if (body.messages?.length) { + body.messages.forEach((item) => { + transcript.push({ + role: item.role ?? 'user', + content: + mapInputContentToTranscriptContent(item.content) ?? + stringifyContent(item.content), + }); + }); + } else if (typeof body.input === 'string') { + transcript.push({ role: 'user', content: body.input }); + } else if (Array.isArray(body.input)) { + body.input.forEach((item) => { + if (item.type === 'additional_tools') return; + + const message = mapInputItemToMessage(item); + + if (!message) { + return; + } + + // A reasoning item yields a reasoning-only entry. Fold it into the next + // assistant message so the upstream sees the reasoning where it belongs + // — attached to the turn that produced it — instead of as a bare turn. + // Anything left unconsumed at the end is dropped: reasoning with no + // following assistant message has nothing to attach to. + if (message.reasoning && !message.content && !message.tool_calls) { + pendingReasoning += message.reasoning; + return; + } + + // Attach any reasoning carried forward from a preceding reasoning item. + // `message.reasoning` is only ever set by the mapper below — clients + // cannot send it, since `ResponsesInputItem` has no such field — so + // there is no pre-existing value to merge with. + if (pendingReasoning) { + message.reasoning = pendingReasoning; + pendingReasoning = ''; + } + + transcript.push(message); + }); + } + + return { + defaults, + model, + transcript, + previousResponseId, + }; +}; + +export const toResponsesUsageNumber = (value: unknown): number => { + const numeric = + typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); + + if (!Number.isFinite(numeric) || numeric < 0) { + return 0; + } + + return numeric; +}; + +export const mapChatUsageToResponses = ( + usage: unknown, +): Record => { + if (!usage || typeof usage !== 'object') { + return { + input_tokens: 0, + input_tokens_details: { cached_tokens: 0 }, + output_tokens: 0, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 0, + }; + } + + const value = usage as { + cache_creation_input_tokens?: unknown; + cache_read_input_tokens?: unknown; + completion_tokens?: unknown; + completion_tokens_details?: { reasoning_tokens?: unknown }; + completion_thinking_tokens?: unknown; + input_tokens_details?: { cached_tokens?: unknown }; + prompt_cache_hit_tokens?: unknown; + prompt_cache_miss_tokens?: unknown; + prompt_cache_write_tokens?: unknown; + prompt_tokens?: unknown; + prompt_tokens_details?: { + cache_creation_tokens?: unknown; + cached_tokens?: unknown; + }; + total_tokens?: unknown; + }; + const outputTokens = toResponsesUsageNumber(value.completion_tokens); + const cachedTokens = toResponsesUsageNumber( + value.prompt_tokens_details?.cached_tokens ?? + value.input_tokens_details?.cached_tokens ?? + value.cache_read_input_tokens ?? + value.prompt_cache_hit_tokens, + ); + const cacheCreationTokens = toResponsesUsageNumber( + value.prompt_tokens_details?.cache_creation_tokens ?? + value.cache_creation_input_tokens ?? + value.prompt_cache_write_tokens, + ); + const reasoningTokens = toResponsesUsageNumber( + value.completion_tokens_details?.reasoning_tokens ?? + value.completion_thinking_tokens, + ); + // Chat usage is the single source of truth for both shapes. Keep the + // Responses counters faithful to it so clients never see zeroed metrics. + // prompt_tokens already covers its cached and created subsets, so the + // split counters are only summed when prompt_tokens is missing. Otherwise + // cached tokens would exceed the reported input total. + const inputTokens = toResponsesUsageNumber( + value.prompt_tokens ?? + toResponsesUsageNumber(value.prompt_cache_miss_tokens) + + cachedTokens + + cacheCreationTokens, + ); + + return { + input_tokens: inputTokens, + input_tokens_details: { cached_tokens: cachedTokens }, + output_tokens: outputTokens, + output_tokens_details: { reasoning_tokens: reasoningTokens }, + total_tokens: + toResponsesUsageNumber(value.total_tokens) || inputTokens + outputTokens, + }; +}; diff --git a/lib/server/proxy/responses-types.ts b/lib/server/proxy/responses-types.ts new file mode 100644 index 0000000..83632de --- /dev/null +++ b/lib/server/proxy/responses-types.ts @@ -0,0 +1,166 @@ +// --------------------------------------------------------------------------- +// OpenAI Responses API types +// --------------------------------------------------------------------------- + +export interface ResponsesInputItem { + type?: string; + role?: string; + content?: unknown; + text?: string; + arguments?: string; + output?: unknown; + name?: string; + call_id?: string; + tools?: Array<{ type?: string; name?: string } & Record>; + /** + * Present on `reasoning` items a client replays from an earlier response. + * We put the reasoning here verbatim; clients echo it back untouched. + * A compaction item carries the same field. + */ + encrypted_content?: string; + /** + * Reasoning summaries. The Agents SDK sends these back as + * `summary: [{type: 'summary_text', text}]`. + */ + summary?: unknown; +} + +export interface SupportedChatTool { + chatName: string; + kind: 'custom' | 'function' | 'mcp' | 'tool_search'; + namespace?: string; + originalName: string; + /** + * True when the client declared this as a provider-executed server tool + * (`web_search_20260209`, `web_fetch_20250910`, `web_search_preview`) rather + * than as its own function. + * + * Translation turns both into ordinary functions for upstream, so without this + * the proxy cannot tell them apart later — and the difference decides whether a + * tool that cannot be executed is dropped or forwarded. + */ + serverDeclared?: boolean; + serverLabel?: string; + tool: Record; +} + +export interface ResponsesRequestBody { + model?: string; + input?: string | ResponsesInputItem[]; + instructions?: string; + messages?: Array<{ role?: string; content?: unknown }>; + stream?: boolean; + metadata?: Record; + reasoning?: Record; + thinking?: Record; + tools?: Array<{ type?: string; name?: string } & Record>; + tool_choice?: unknown; + max_output_tokens?: number; + previous_response_id?: string; +} + +export type ResponseSessionDefaults = Pick< + ResponsesRequestBody, + 'instructions' | 'metadata' | 'tools' | 'tool_choice' +>; + +export interface ResponseSession { + accessKeyId: string | null; + credentialFilename: string | null; + createdAt: number; + id: string; + model: string; + transcript: TranscriptMessage[]; + defaults: ResponseSessionDefaults; + upstreamProtocol?: 'chat' | 'responses'; +} + +export interface ChatResponseToolCall { + index?: number; + id?: string; + type?: string; + function?: { + arguments?: string; + name?: string; + }; +} + +export interface ChatResponseMessage { + content?: unknown; + tool_calls?: ChatResponseToolCall[]; + /** Reasoning the upstream produced alongside `content`. */ + reasoning_content?: string; + reasoning?: string; +} + +export interface ChatImagePart { + image_url: { url: string }; + type: 'image_url'; +} + +export interface ChatTextPart { + text: string; + type: 'text'; +} + +export type ChatContentPart = string | ChatTextPart | ChatImagePart; + +/** + * Transcript content. Images are kept as structured parts so they survive the + * Chat-shaped round trip through the transcript and reach the model as images + * instead of a JSON dump. + */ +export type TranscriptContent = string | ChatContentPart[]; + +export interface TranscriptMessage { + role: string; + content: TranscriptContent | null; + tool_calls?: Array<{ + id: string; + type: string; + function: { + name: string; + arguments: string; + }; + }>; + tool_call_id?: string; + /** + * Prior-turn reasoning recovered from a replayed reasoning item. + * + * Carried on the assistant message the reasoning belongs to rather than sent + * as its own message: the chat upstream has no standalone reasoning entry, + * and a reasoning-only message would be an empty turn. + */ + reasoning?: string; +} + +export interface StreamingToolCallState { + addedEmitted: boolean; + arguments: string; + canonicalKey: string; + callId: string; + name: string; + outputIndex: number; + outputItemId: string; + pendingArgumentDeltas: string[]; +} + +export interface StreamingMessageState { + outputIndex: number | null; + outputItemId: string; +} + +export interface ResponsesServerToolItem { + completed: Record; + inProgress: Record; + outputIndex: number; +} + +export interface ResponseSessionMetadata { + bytes: number; + createdAt: number; +} + +export type SupportedResponsesTool = NonNullable< + ResponsesRequestBody['tools'] +>[number]; diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 9b7f5b5..5a8ec2c 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -1,2718 +1,47 @@ -import type { NextRequest } from 'next/server'; - -import { - getDefaultModel, - isWebFetchEnabled, - isWebSearchEnabled, -} from '../domain/config'; -import { stringifyContent } from '../shared/content'; -import { - createSseResponse, - DONE_FRAME_TEXT, - encodeDoneFrame, - eventFrameText, -} from '../shared/sse'; -import { getCredentialSupportedModels } from '../domain/credentials'; -import type { DebugTrace } from '../domain/debug'; -import { - buildWebFetchToolDefinition, - buildWebSearchToolDefinition, - markServerTool, - normalizeToolName, - WEB_FETCH_TOOL_NAME, - WEB_FETCH_TOOL_TYPE_PREFIX, - WEB_SEARCH_TOOL_NAME, - WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; -import { - buildImageGenerationChatTool, - buildResponsesImageGenerationCallItem, - executeImageGenerationLoop, - IMAGE_GENERATION_CHAT_TOOL_NAME, - IMAGE_GENERATION_TOOL_TYPE, - type ImageGenerationExecution, -} from './image-generation'; -import { - extractImageUrl, - isImageContentPart, - proxyChatCompletions, - proxyResponsesUpstream, - resolveProxyContext, - resolveProxyContextByCredentialFilename, - type ProxyContext, -} from './codebuddy'; -import { - getServerToolExecutions, - type ServerToolExecution, - type ServerToolInvocation, -} from './web-search-loop'; -import { resolveRequestAccessKey } from './auth'; -import { createErrorResponse } from '../shared/http'; -import { - createStreamCloser, - readTimeoutFrame, - responsesStreamErrorChunks, - toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { - deleteStorageJson, - getStorageBackendMeta, - listStorageJson, - readStorageJson, - writeStorageJson, -} from '../storage'; - -interface ResponsesInputItem { - type?: string; - role?: string; - content?: unknown; - text?: string; - arguments?: string; - output?: unknown; - name?: string; - call_id?: string; - tools?: Array<{ type?: string; name?: string } & Record>; - /** - * Present on `reasoning` items a client replays from an earlier response. - * We put the reasoning here verbatim; clients echo it back untouched. - * A compaction item carries the same field. - */ - encrypted_content?: string; - /** - * Reasoning summaries. The Agents SDK sends these back as - * `summary: [{type: 'summary_text', text}]`. - */ - summary?: unknown; -} - -interface SupportedChatTool { - chatName: string; - kind: 'custom' | 'function' | 'mcp' | 'tool_search'; - namespace?: string; - originalName: string; - /** - * True when the client declared this as a provider-executed server tool - * (`web_search_20260209`, `web_fetch_20250910`, `web_search_preview`) rather - * than as its own function. - * - * Translation turns both into ordinary functions for upstream, so without this - * the proxy cannot tell them apart later — and the difference decides whether a - * tool that cannot be executed is dropped or forwarded. - */ - serverDeclared?: boolean; - serverLabel?: string; - tool: Record; -} - -interface ResponsesRequestBody { - model?: string; - input?: string | ResponsesInputItem[]; - instructions?: string; - messages?: Array<{ role?: string; content?: unknown }>; - stream?: boolean; - metadata?: Record; - reasoning?: Record; - thinking?: Record; - tools?: Array<{ type?: string; name?: string } & Record>; - tool_choice?: unknown; - max_output_tokens?: number; - previous_response_id?: string; -} - -type ResponseSessionDefaults = Pick< - ResponsesRequestBody, - 'instructions' | 'metadata' | 'tools' | 'tool_choice' ->; - -interface ResponseSession { - accessKeyId: string | null; - credentialFilename: string | null; - createdAt: number; - id: string; - model: string; - transcript: TranscriptMessage[]; - defaults: ResponseSessionDefaults; - upstreamProtocol?: 'chat' | 'responses'; -} - -interface ChatResponseToolCall { - index?: number; - id?: string; - type?: string; - function?: { - arguments?: string; - name?: string; - }; -} - -interface ChatResponseMessage { - content?: unknown; - tool_calls?: ChatResponseToolCall[]; - /** Reasoning the upstream produced alongside `content`. */ - reasoning_content?: string; - reasoning?: string; -} - -interface ChatImagePart { - image_url: { url: string }; - type: 'image_url'; -} - -interface ChatTextPart { - text: string; - type: 'text'; -} - -type ChatContentPart = string | ChatTextPart | ChatImagePart; - -/** - * Transcript content. Images are kept as structured parts so they survive the - * Chat-shaped round trip through the transcript and reach the model as images - * instead of a JSON dump. - */ -type TranscriptContent = string | ChatContentPart[]; - -interface TranscriptMessage { - role: string; - content: TranscriptContent | null; - tool_calls?: Array<{ - id: string; - type: string; - function: { - name: string; - arguments: string; - }; - }>; - tool_call_id?: string; - /** - * Prior-turn reasoning recovered from a replayed reasoning item. - * - * Carried on the assistant message the reasoning belongs to rather than sent - * as its own message: the chat upstream has no standalone reasoning entry, - * and a reasoning-only message would be an empty turn. - */ - reasoning?: string; -} - -interface StreamingToolCallState { - addedEmitted: boolean; - arguments: string; - canonicalKey: string; - callId: string; - name: string; - outputIndex: number; - outputItemId: string; - pendingArgumentDeltas: string[]; -} - -interface StreamingMessageState { - outputIndex: number | null; - outputItemId: string; -} - -interface ResponsesServerToolItem { - completed: Record; - inProgress: Record; - outputIndex: number; -} - -interface ResponseSessionMetadata { - bytes: number; - createdAt: number; -} - -type SupportedResponsesTool = NonNullable< - ResponsesRequestBody['tools'] ->[number]; - -const TOOL_SEARCH_PROXY_NAME = 'tool_search'; -const CUSTOM_TOOL_INPUT_FIELD = 'input'; -const CUSTOM_TOOL_INPUT_DESCRIPTION = - 'Raw string input for the original custom tool.'; -const MAX_RESPONSE_SESSIONS = 1_000; -const RESPONSE_SESSION_TTL_MS = 60 * 60 * 1000; -const MAX_RESPONSE_SESSION_BYTES = 8 * 1024 * 1024; -const MAX_RESPONSE_SESSION_TOTAL_BYTES = 64 * 1024 * 1024; -const MAX_RESPONSE_TRANSCRIPT_MESSAGES = 200; -const RESPONSE_SESSION_NAMESPACE = 'responses'; -const RESPONSE_SESSION_INDEX_NAMESPACE = 'response-session-index'; -const MAX_STREAM_BUFFER_LENGTH = 1_000_000; -const MAX_STREAM_TEXT_LENGTH = 2_000_000; -const MAX_TOOL_ARGUMENT_LENGTH = 1_000_000; -const MAX_TOOL_NAME_LENGTH = 256; - -const globalResponsesState = globalThis as typeof globalThis & { - __codebuddy2apiResponseSessions__?: Map; - __codebuddy2apiResponseSessionBytes__?: Map; - __codebuddy2apiResponseSessionTotalBytes__?: number; -}; - -const getSessionStore = (): Map => { - if (!globalResponsesState.__codebuddy2apiResponseSessions__) { - globalResponsesState.__codebuddy2apiResponseSessions__ = new Map(); - } - - return globalResponsesState.__codebuddy2apiResponseSessions__; -}; - -const getSessionByteStore = (): Map => { - if (!globalResponsesState.__codebuddy2apiResponseSessionBytes__) { - globalResponsesState.__codebuddy2apiResponseSessionBytes__ = new Map(); - } - - return globalResponsesState.__codebuddy2apiResponseSessionBytes__; -}; - -const getSessionTotalBytes = (): number => { - return globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ ?? 0; -}; - -const setSessionTotalBytes = (value: number): void => { - globalResponsesState.__codebuddy2apiResponseSessionTotalBytes__ = value; -}; - -const removeLocalResponseSession = (id: string): void => { - const byteStore = getSessionByteStore(); - const store = getSessionStore(); - const bytes = byteStore.get(id) ?? 0; - store.delete(id); - byteStore.delete(id); - setSessionTotalBytes(Math.max(0, getSessionTotalBytes() - bytes)); -}; - -const pruneResponseSessions = (): void => { - const store = getSessionStore(); - const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; - - for (const [id, session] of store) { - if (session.createdAt <= expiresBefore) { - removeLocalResponseSession(id); - } - } - - while ( - store.size > MAX_RESPONSE_SESSIONS || - getSessionTotalBytes() > MAX_RESPONSE_SESSION_TOTAL_BYTES - ) { - const oldestId = store.keys().next().value; - - // Guard against a byte total that has drifted out of step with the map. - // Without this, an empty map with a positive total makes the removal a - // no-op and spins here forever, blocking the event loop. - if (oldestId === undefined) { - setSessionTotalBytes(0); - break; - } - - removeLocalResponseSession(oldestId); - } -}; - -const prunePgResponseSessions = async (): Promise => { - const metadataDocuments = await listStorageJson( - RESPONSE_SESSION_INDEX_NAMESPACE, - ); - const expiresBefore = Date.now() - RESPONSE_SESSION_TTL_MS; - const candidates = metadataDocuments - .map((document) => ({ key: document.key, ...document.value })) - .sort((left, right) => left.createdAt - right.createdAt); - const toDelete = candidates.filter( - (candidate) => candidate.createdAt <= expiresBefore, - ); - const remaining = candidates.filter( - (candidate) => candidate.createdAt > expiresBefore, - ); - let totalBytes = remaining.reduce( - (total, candidate) => total + candidate.bytes, - 0, - ); - - while ( - remaining.length > MAX_RESPONSE_SESSIONS || - totalBytes > MAX_RESPONSE_SESSION_TOTAL_BYTES - ) { - const candidate = remaining.shift()!; - toDelete.push(candidate); - totalBytes -= candidate.bytes; - } - - await Promise.all( - toDelete.flatMap((candidate) => [ - deleteStorageJson(RESPONSE_SESSION_NAMESPACE, candidate.key), - deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, candidate.key), - ]), - ); -}; - -const isPgResponseSessionStore = (): boolean => { - return getStorageBackendMeta().backend === 'pg'; -}; - -const getResponseSession = async ( - id: string, -): Promise => { - if (isPgResponseSessionStore()) { - const session = await readStorageJson( - RESPONSE_SESSION_NAMESPACE, - id, - ); - if (!session || session.createdAt <= Date.now() - RESPONSE_SESSION_TTL_MS) { - if (session) { - await deleteStorageJson(RESPONSE_SESSION_NAMESPACE, id); - await deleteStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, id); - } - return undefined; - } - return session; - } - - pruneResponseSessions(); - return getSessionStore().get(id); -}; - -const getValidatedPreviousSession = async ( - previousResponseId: string | null, - accessKeyId: string | null, -): Promise => { - const previousSession = previousResponseId - ? await getResponseSession(previousResponseId) - : undefined; - - if ( - previousResponseId && - (!previousSession || previousSession.accessKeyId !== accessKeyId) - ) { - throw new Error('Unknown or expired previous_response_id'); - } - - return previousSession; -}; - -const storeResponseSession = async ( - session: ResponseSession, -): Promise => { - const serialized = JSON.stringify(session); - if (Buffer.byteLength(serialized, 'utf8') > MAX_RESPONSE_SESSION_BYTES) { - throw new Error('Response session exceeds the maximum size'); - } - - if (isPgResponseSessionStore()) { - await writeStorageJson(RESPONSE_SESSION_NAMESPACE, session.id, session); - await writeStorageJson(RESPONSE_SESSION_INDEX_NAMESPACE, session.id, { - bytes: Buffer.byteLength(serialized, 'utf8'), - createdAt: session.createdAt, - }); - try { - await prunePgResponseSessions(); - } catch (error) { - console.warn('[CodeBuddy2API] Unable to prune Responses sessions', error); - } - return; - } - - const store = getSessionStore(); - const byteStore = getSessionByteStore(); - const previousBytes = byteStore.get(session.id) ?? 0; - const sessionBytes = Buffer.byteLength(serialized, 'utf8'); - store.set(session.id, session); - byteStore.set(session.id, sessionBytes); - setSessionTotalBytes(getSessionTotalBytes() - previousBytes + sessionBytes); - pruneResponseSessions(); -}; - -const storeUpstreamResponseBinding = async ({ - model, - proxyContext, - responseId, -}: { - model: string; - proxyContext: ProxyContext; - responseId: string; -}): Promise => { - await storeResponseSession({ - accessKeyId: proxyContext.accessKeyId, - credentialFilename: proxyContext.credentialFilename, - createdAt: Date.now(), - defaults: {}, - id: responseId, - model, - transcript: [], - upstreamProtocol: 'responses', - }); -}; - -const flattenNamespaceToolName = (namespace: string, name: string): string => { - return `${namespace}__${name}`; -}; - -const extractFunctionDefinition = ( - tool: Record, -): Record | null => { - const nested = - typeof tool.function === 'object' && tool.function !== null - ? (tool.function as Record) - : {}; - - const name = nested.name ?? tool.name; - if (typeof name !== 'string' || name.length === 0) { - return null; - } - - const functionDef: Record = { name }; - - const description = nested.description ?? tool.description; - if (description !== undefined) { - functionDef.description = description; - } - - const parameters = nested.parameters ?? tool.parameters; - if (parameters !== undefined) { - functionDef.parameters = parameters; - } - - const strict = nested.strict ?? tool.strict; - if (strict !== undefined) { - functionDef.strict = strict; - } - - return functionDef; -}; - -const buildCustomToolDefinition = ( - tool: Record, -): Record | null => { - const name = typeof tool.name === 'string' ? tool.name.trim() : ''; - - if (!name) { - return null; - } - - const description = - typeof tool.description === 'string' && tool.description.trim() - ? tool.description - : `Custom tool ${name}`; - - return { - name, - description, - parameters: { - type: 'object', - properties: { - [CUSTOM_TOOL_INPUT_FIELD]: { - type: 'string', - description: CUSTOM_TOOL_INPUT_DESCRIPTION, - }, - }, - required: [CUSTOM_TOOL_INPUT_FIELD], - }, - }; -}; - -const buildToolSearchDefinition = (): Record => { - return { - name: TOOL_SEARCH_PROXY_NAME, - description: - 'Search and load Codex tools, plugins, connectors, and MCP namespaces for the current task.', - parameters: { - type: 'object', - properties: { - query: { - type: 'string', - description: 'Search query for tools or connectors to load.', - }, - limit: { - type: 'integer', - description: 'Maximum number of tool groups to return.', - }, - }, - required: ['query'], - }, - }; -}; - -const toSupportedChatTool = ( - tool: SupportedResponsesTool, - namespace?: string, -): SupportedChatTool[] => { - const toolType = typeof tool.type === 'string' ? tool.type : 'function'; - - // Server-side search and fetch carry no function schema, so the generic - // branch below drops them. Emit them as functions unconditionally and let - // the proxy loop resolve the configured backend asynchronously. SearXNG - // needs local configuration, while CodeBuddy search does not. - if ( - normalizeToolName(toolType).startsWith( - normalizeToolName(WEB_SEARCH_TOOL_TYPE_PREFIX), - ) - ) { - const definition = buildWebSearchToolDefinition(); - - return [ - { - chatName: WEB_SEARCH_TOOL_NAME, - kind: 'function', - originalName: WEB_SEARCH_TOOL_NAME, - serverDeclared: true, - tool: definition, - }, - ]; - } - - // Fetch needs no deployment-level configuration — the local backend is always - // available and the CodeBuddy backend needs only a credential — so it is - // advertised unconditionally and gated later by the enable toggle. - if ( - normalizeToolName(toolType).startsWith( - normalizeToolName(WEB_FETCH_TOOL_TYPE_PREFIX), - ) - ) { - const definition = buildWebFetchToolDefinition(); - - return [ - { - chatName: WEB_FETCH_TOOL_NAME, - kind: 'function', - originalName: WEB_FETCH_TOOL_NAME, - serverDeclared: true, - tool: definition, - }, - ]; - } - - // Image generation has no chat-protocol equivalent, so the declaration is - // rewritten as a function and the call is executed by the proxy. It is - // advertised only on the chat path: the responses passthrough hands the - // native declaration straight to the upstream, which supports it. - if (toolType === IMAGE_GENERATION_TOOL_TYPE) { - return [ - { - chatName: IMAGE_GENERATION_CHAT_TOOL_NAME, - kind: 'function' as const, - originalName: IMAGE_GENERATION_TOOL_TYPE, - serverDeclared: true, - tool: buildImageGenerationChatTool(), - }, - ]; - } - - if (toolType === 'namespace') { - const namespaceName = typeof tool.name === 'string' ? tool.name.trim() : ''; - const children = ( - Array.isArray(tool.tools) - ? tool.tools - : Array.isArray(tool.children) - ? tool.children - : [] - ).filter((item): item is SupportedResponsesTool => { - return Boolean(item && typeof item === 'object'); - }); - - if (!namespaceName || !children.length) { - return []; - } - - return children.flatMap((child) => - toSupportedChatTool(child, namespaceName), - ); - } - - if (toolType === 'tool_search') { - const definition = buildToolSearchDefinition(); - return [ - { - chatName: TOOL_SEARCH_PROXY_NAME, - kind: 'tool_search', - originalName: TOOL_SEARCH_PROXY_NAME, - tool: definition, - }, - ]; - } - - if (toolType === 'custom') { - const definition = buildCustomToolDefinition(tool); - - if (!definition || typeof definition.name !== 'string') { - return []; - } - - return [ - { - chatName: definition.name, - kind: 'custom', - originalName: definition.name, - tool: definition, - }, - ]; - } - - const functionDef = extractFunctionDefinition(tool); - - if (!functionDef || typeof functionDef.name !== 'string') { - return []; - } - - const originalName = functionDef.name; - const chatName = namespace - ? flattenNamespaceToolName(namespace, originalName) - : toolType === 'mcp' && - typeof tool.server_label === 'string' && - tool.server_label.trim() - ? flattenNamespaceToolName(tool.server_label.trim(), originalName) - : originalName; - - return [ - { - chatName, - kind: toolType === 'mcp' ? 'mcp' : 'function', - namespace: - namespace || - (typeof tool.server_label === 'string' ? tool.server_label : undefined), - originalName, - serverLabel: - toolType === 'mcp' && typeof tool.server_label === 'string' - ? tool.server_label - : undefined, - tool: { - ...functionDef, - name: chatName, - }, - }, - ]; -}; - -/** - * True when the client declared an `image_generation` tool. Gating on this - * keeps the ordinary path free of an extra upstream round trip. - */ -const hasImageGenerationTool = ( - tools: ResponsesRequestBody['tools'], -): boolean => { - return Boolean( - tools?.some( - (tool) => - typeof tool?.type === 'string' && - tool.type.toLowerCase().replaceAll('-', '_') === - IMAGE_GENERATION_TOOL_TYPE, - ), - ); -}; - -const getSupportedChatTools = ( - tools: ResponsesRequestBody['tools'], -): SupportedChatTool[] => { - if (!tools?.length) { - return []; - } - - return tools.flatMap((tool) => toSupportedChatTool(tool)); -}; - -const findSupportedToolByName = ( - tools: ResponsesRequestBody['tools'], - name: string, -): SupportedChatTool | null => { - if (!tools?.length || !name) { - return null; - } - - return ( - getSupportedChatTools(tools).find( - (tool) => tool.chatName === name || tool.originalName === name, - ) ?? null - ); -}; - -const hasSupportedLongerToolNamePrefix = ( - tools: ResponsesRequestBody['tools'], - prefix: string, -): boolean => { - if (!tools?.length || !prefix) { - return false; - } - - return getSupportedChatTools(tools).some((tool) => { - const name = tool.chatName; - return ( - typeof name === 'string' && - name.length > prefix.length && - name.startsWith(prefix) - ); - }); -}; - -const buildResponsesToolCallOutputItem = ( - tools: ResponsesRequestBody['tools'], - toolCall: { - arguments: string; - callId: string; - id: string; - name: string; - status: 'completed' | 'in_progress'; - }, -): Record => { - const originalTool = findSupportedToolByName(tools, toolCall.name); - const itemType = originalTool?.kind === 'mcp' ? 'mcp_call' : 'function_call'; - const item: Record = { - id: toolCall.id, - type: itemType, - call_id: toolCall.callId, - name: originalTool?.originalName ?? toolCall.name ?? 'function', - arguments: toolCall.arguments, - status: toolCall.status, - }; - - if (originalTool?.kind === 'mcp' && originalTool.serverLabel) { - item.server_label = originalTool.serverLabel; - } - - if (originalTool?.kind === 'function' && originalTool.namespace) { - item.namespace = originalTool.namespace; - } - - return item; -}; - -const getResponsesToolCallArgumentDeltaEventType = ( - tools: ResponsesRequestBody['tools'], - name: string, -): - | 'response.function_call_arguments.delta' - | 'response.mcp_call_arguments.delta' => { - return findSupportedToolByName(tools, name)?.kind === 'mcp' - ? 'response.mcp_call_arguments.delta' - : 'response.function_call_arguments.delta'; -}; - -const buildAssistantTranscriptToolCalls = ( - toolCalls: ChatResponseToolCall[], - tools?: ResponsesRequestBody['tools'], -): TranscriptMessage['tool_calls'] | undefined => { - if (!toolCalls.length) { - return undefined; - } - - return toolCalls.map((toolCall, index) => ({ - id: normalizeToolCallId(toolCall.id, index), - type: 'function', - function: { - name: - findSupportedToolByName(tools, toolCall.function?.name ?? '') - ?.originalName ?? - toolCall.function?.name ?? - 'function', - arguments: toolCall.function?.arguments ?? '', - }, - })); -}; - -const buildStreamingAssistantTranscriptToolCalls = ( - toolCallStates: StreamingToolCallState[], - tools?: ResponsesRequestBody['tools'], -): TranscriptMessage['tool_calls'] | undefined => { - if (!toolCallStates.length) { - return undefined; - } - - return toolCallStates.map((toolCallState) => ({ - id: toolCallState.callId, - type: 'function', - function: { - arguments: toolCallState.arguments, - name: - findSupportedToolByName(tools, toolCallState.name)?.originalName ?? - toolCallState.name, - }, - })); -}; - -const getAssistantTranscriptContent = ( - outputText: string, - toolCalls: TranscriptMessage['tool_calls'] | undefined, -): string | null => { - return toolCalls?.length ? outputText || null : outputText; -}; - -/** - * Keeps image parts as structured content so the chat path can rebuild them - * upstream. Text parts are still flattened: the transcript is persisted across - * turns and replayed as Chat messages, and the Responses converter only - * recognises images in the OpenAI `image_url` shape. - */ -const mapInputContentToTranscriptContent = ( - content: unknown, -): TranscriptContent | null => { - if (typeof content === 'string') { - return content; - } - - if (!Array.isArray(content)) { - return null; - } - - const parts = content.filter((part) => part !== null && part !== undefined); - - if (!parts.some(isImageContentPart)) { - return null; - } - - const mapped = parts.flatMap((part): ChatContentPart[] => { - if (typeof part === 'string') { - return [part]; - } - - if (isImageContentPart(part)) { - const imageUrl = extractImageUrl(part); - - return imageUrl - ? [{ image_url: { url: imageUrl }, type: 'image_url' }] - : []; - } - - if (part && typeof part === 'object' && 'text' in part) { - return [String((part as { text?: unknown }).text ?? '')]; - } - - return []; - }); - - return mapped.length ? mapped : null; -}; - -/** - * Marks an `encrypted_content` value we minted, so we can tell it apart from a - * blob issued by someone else. - * - * Not a security measure. Codex never opens this field — it only echoes it — so - * plaintext round-trips fine, but a marker is what stops us from reading a - * genuinely encrypted blob as if it were reasoning text. - */ -const REASONING_PREFIX = 'cbreason1:'; - -/** - * Pulls readable reasoning out of a replayed `reasoning` item. - * - * Only values we minted are used: anything else — an OpenAI-issued blob, say — - * is opaque ciphertext, and forwarding it upstream would send gibberish where - * reasoning belongs. The summary is the fallback in that case. - * - * The Agents SDK sends summaries as `summary: [{type: 'summary_text', text}]`, - * so a client that never received our blob still gets its reasoning through. - */ -const extractReasoningFromItem = (item: ResponsesInputItem): string => { - const blob = item.encrypted_content; - - if (typeof blob === 'string' && blob.startsWith(REASONING_PREFIX)) { - return blob.slice(REASONING_PREFIX.length); - } - - if (!Array.isArray(item.summary)) { - return ''; - } - - return item.summary - .map((entry) => { - if (typeof entry === 'string') { - return entry; - } - - if (entry && typeof entry === 'object' && 'text' in entry) { - return String((entry as { text?: unknown }).text ?? ''); - } - - return ''; - }) - .join(''); -}; - -const mapInputItemToMessage = ( - item: ResponsesInputItem, -): TranscriptMessage | null => { - if (item.type === 'reasoning' || item.type === 'compaction') { - // Reasoning is not a message. Without this branch the item fell through to - // the plain-message case at the bottom, where it has neither `role` nor - // `content` — becoming an empty `{role:'user', content:''}` entry that the - // chat upstream sees as a turn the user never sent, repeated on every - // later turn of the conversation. - // - // Signal the reasoning back to the caller instead, which attaches it to the - // assistant message it accompanies. Returning `null` when there is nothing - // to recover keeps an empty reasoning item from emitting a message at all. - const reasoning = extractReasoningFromItem(item); - - return reasoning ? { role: 'assistant', content: null, reasoning } : null; - } - - if (item.type === 'function_call' || item.type === 'mcp_call') { - return { - role: 'assistant', - content: null, - tool_calls: [ - { - id: item.call_id ?? createResponseOutputId(), - type: 'function', - function: { - name: item.name ?? 'function', - arguments: item.arguments ?? '', - }, - }, - ], - }; - } - - if (item.type === 'function_call_output' || item.type === 'mcp_call_output') { - // A tool may return an image, e.g. a screenshot. Keep it structured so the - // Responses converter can rebuild it as an image; stringifying would hand - // the model the base64 payload as text. - const outputContent = - mapInputContentToTranscriptContent(item.output) ?? - stringifyContent(item.output); - - if (item.call_id) { - return { - role: 'tool', - content: outputContent, - tool_call_id: item.call_id, - }; - } - - return { - role: 'user', - content: outputContent, - }; - } - - if (item.type === 'mcp_approval_response') { - return { - role: 'user', - content: JSON.stringify(item), - }; - } - - // Every other item type returns above, so what is left is a plain message: - // either one with a declared `type: 'message'`, or one carrying only - // `role`/`content`. Images are kept structured so the chat path can rebuild - // them; a message without an image stays flattened. - const imageContent = mapInputContentToTranscriptContent(item.content); - - if (imageContent !== null) { - return { - role: item.role ?? 'user', - content: imageContent, - }; - } - - return { - role: item.role ?? 'user', - content: item.text ?? stringifyContent(item.content), - }; -}; - -const createResponseId = (): string => { - return `resp_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const createMessageId = (): string => { - return `msg_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const createResponseReasoningId = (): string => { - return `rs_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const createResponseOutputId = (): string => { - return `fc_${crypto.randomUUID().replaceAll('-', '')}`; -}; - -const normalizeToolCallId = (id: string | undefined, index: number): string => { - if (id && !id.startsWith('tooluse_')) { - return id; - } - - return `call_${id?.replace(/^tooluse_/, '') ?? index + 1}`; -}; - -export const translateResponsesToolsToChat = ( - tools: ResponsesRequestBody['tools'], -): unknown[] | undefined => { - if (!tools?.length) { - return undefined; - } - - const supported = getSupportedChatTools(tools); - if (!supported.length) { - return undefined; - } - - return supported.map((tool) => { - return { - type: 'function', - function: tool.tool, - ...(tool.serverDeclared ? markServerTool({}) : {}), - }; - }); -}; - -const translateResponsesToolChoiceToChat = (toolChoice: unknown): unknown => { - if (typeof toolChoice !== 'object' || toolChoice === null) { - return toolChoice; - } - - const choice = toolChoice as Record; - - if ( - choice.type === 'function' && - choice.function && - typeof choice.function === 'object' - ) { - return toolChoice; - } - - if ( - (choice.type === 'auto' || - choice.type === 'none' || - choice.type === 'required') && - typeof choice.type === 'string' - ) { - return choice.type; - } - - // Responses API selects a function by name: - // {type: 'function', name: 'fn'} -> chat schema {type: 'function', function: {name: 'fn'}} - if (typeof choice.name === 'string') { - return { - type: 'function', - function: { name: choice.name }, - }; - } - - return toolChoice; -}; - -const translateResponsesToolChoiceToChatWithTools = ( - tools: ResponsesRequestBody['tools'], - toolChoice: unknown, -): unknown => { - const translated = translateResponsesToolChoiceToChat(toolChoice); - - if (typeof translated !== 'object' || translated === null) { - return translated; - } - - const choice = translated as Record; - - if ( - choice.type === 'function' && - typeof choice.function === 'object' && - choice.function !== null - ) { - const functionChoice = choice.function as Record; - if (typeof functionChoice.name === 'string') { - return { - ...choice, - function: { - ...functionChoice, - name: resolveChatToolName(tools, functionChoice.name), - }, - }; - } - } - - return translated; -}; - -const getNamedToolChoice = (toolChoice: unknown): string | null => { - if (typeof toolChoice !== 'object' || toolChoice === null) { - return null; - } - - const choice = toolChoice as Record; - - if (typeof choice.name === 'string' && choice.name.length > 0) { - return choice.name; - } - - if ( - choice.type === 'function' && - typeof choice.function === 'object' && - choice.function !== null && - typeof (choice.function as Record).name === 'string' - ) { - return (choice.function as Record).name; - } - - return null; -}; - -const getResponsesCompatibilityError = ( - tools: ResponsesRequestBody['tools'], - toolChoice: unknown, -): Response | null => { - const supportedTools = getSupportedChatTools(tools); - - if (toolChoice === 'required' && supportedTools.length === 0) { - return createErrorResponse( - 400, - 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', - ); - } - - if (typeof toolChoice === 'object' && toolChoice !== null) { - const choice = toolChoice as Record; - const isPretranslatedFunctionChoice = - choice.type === 'function' && - typeof choice.function === 'object' && - choice.function !== null; - const isSimpleChoiceType = - choice.type === 'auto' || - choice.type === 'none' || - choice.type === 'required'; - const isNamedFunctionLikeChoice = typeof choice.name === 'string'; - - if ( - !isPretranslatedFunctionChoice && - !isSimpleChoiceType && - !isNamedFunctionLikeChoice - ) { - return createErrorResponse( - 400, - 'Unsupported Responses tool_choice for this /v1/responses adapter', - ); - } - - if (choice.type === 'required' && supportedTools.length === 0) { - return createErrorResponse( - 400, - 'tool_choice=required requires at least one supported tool for this /v1/responses adapter', - ); - } - } - - const namedToolChoice = getNamedToolChoice(toolChoice); - if (namedToolChoice) { - const supportedNames = new Set( - supportedTools - .map((tool) => tool.originalName) - .filter((name): name is string => typeof name === 'string'), - ); - - if (!supportedNames.has(namedToolChoice)) { - return createErrorResponse( - 400, - 'tool_choice references a tool that is not available to this /v1/responses adapter', - ); - } - } - - return null; -}; - -const getStreamingToolCallCanonicalKey = ( - toolCall: ChatResponseToolCall, - position: number, -): string => { - if (toolCall.id) { - return `id:${toolCall.id}`; - } - - if (typeof toolCall.index === 'number') { - return `index:${toolCall.index}`; - } - - return `position:${position}`; -}; - -const getStreamingToolCallLookupKeys = ( - toolCall: ChatResponseToolCall, - position: number, -): string[] => { - if (toolCall.id || typeof toolCall.index === 'number') { - return [ - toolCall.id ? `id:${toolCall.id}` : null, - typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, - ].filter((key): key is string => key !== null); - } - - return [`position:${position}`]; -}; - -const prepareTranscript = async ( - body: ResponsesRequestBody, - accessKeyId: string | null, - previousSession?: ResponseSession, -): Promise<{ - defaults: ResponseSessionDefaults; - model: string; - transcript: TranscriptMessage[]; - previousResponseId: string | null; -}> => { - const previousResponseId = body.previous_response_id ?? null; - const resolvedPreviousSession = - previousSession ?? - (await getValidatedPreviousSession(previousResponseId, accessKeyId)); - - const transcript = (resolvedPreviousSession?.transcript ?? []).slice( - -MAX_RESPONSE_TRANSCRIPT_MESSAGES, - ); - // Reasoning recovered from replayed reasoning items, awaiting the assistant - // message it belongs to. Declared here so it spans the whole input array. - let pendingReasoning = ''; - while (transcript[0]?.role === 'tool') { - transcript.shift(); - } - const model = - typeof body.model === 'string' && body.model.trim() - ? body.model - : (resolvedPreviousSession?.model ?? (await getDefaultModel())); - const additionalTools = Array.isArray(body.input) - ? body.input.flatMap((item) => - item?.type === 'additional_tools' && Array.isArray(item.tools) - ? item.tools - : [], - ) - : []; - const baseTools = body.tools ?? resolvedPreviousSession?.defaults.tools; - const requestTools = [...(baseTools ?? []), ...additionalTools]; - const defaults = { - instructions: - body.instructions ?? - resolvedPreviousSession?.defaults.instructions ?? - undefined, - metadata: - body.metadata ?? resolvedPreviousSession?.defaults.metadata ?? undefined, - tools: requestTools.length > 0 ? requestTools : baseTools, - tool_choice: - body.tool_choice ?? - resolvedPreviousSession?.defaults.tool_choice ?? - undefined, - }; - - if (body.messages?.length) { - body.messages.forEach((item) => { - transcript.push({ - role: item.role ?? 'user', - content: - mapInputContentToTranscriptContent(item.content) ?? - stringifyContent(item.content), - }); - }); - } else if (typeof body.input === 'string') { - transcript.push({ role: 'user', content: body.input }); - } else if (Array.isArray(body.input)) { - body.input.forEach((item) => { - if (item.type === 'additional_tools') return; - - const message = mapInputItemToMessage(item); - - if (!message) { - return; - } - - // A reasoning item yields a reasoning-only entry. Fold it into the next - // assistant message so the upstream sees the reasoning where it belongs - // — attached to the turn that produced it — instead of as a bare turn. - // Anything left unconsumed at the end is dropped: reasoning with no - // following assistant message has nothing to attach to. - if (message.reasoning && !message.content && !message.tool_calls) { - pendingReasoning += message.reasoning; - return; - } - - // Attach any reasoning carried forward from a preceding reasoning item. - // `message.reasoning` is only ever set by the mapper below — clients - // cannot send it, since `ResponsesInputItem` has no such field — so - // there is no pre-existing value to merge with. - if (pendingReasoning) { - message.reasoning = pendingReasoning; - pendingReasoning = ''; - } - - transcript.push(message); - }); - } - - return { - defaults, - model, - transcript, - previousResponseId, - }; -}; - -const resolveChatToolName = ( - tools: ResponsesRequestBody['tools'], - name: string, -): string => { - return findSupportedToolByName(tools, name)?.chatName ?? name; -}; - -const normalizeTranscriptMessageToolNames = ( - transcript: TranscriptMessage[], - tools: ResponsesRequestBody['tools'], -): TranscriptMessage[] => { - return transcript.map((message) => { - if (!message.tool_calls?.length) { - return message; - } - - return { - ...message, - tool_calls: message.tool_calls.map((toolCall) => ({ - ...toolCall, - function: { - ...toolCall.function, - name: resolveChatToolName(tools, toolCall.function.name), - }, - })), - }; - }); -}; - -const toResponsesUsageNumber = (value: unknown): number => { - const numeric = - typeof value === 'number' ? value : Number.parseFloat(String(value ?? '')); - - if (!Number.isFinite(numeric) || numeric < 0) { - return 0; - } - - return numeric; -}; - -const mapChatUsageToResponses = (usage: unknown): Record => { - if (!usage || typeof usage !== 'object') { - return { - input_tokens: 0, - input_tokens_details: { cached_tokens: 0 }, - output_tokens: 0, - output_tokens_details: { reasoning_tokens: 0 }, - total_tokens: 0, - }; - } - - const value = usage as { - cache_creation_input_tokens?: unknown; - cache_read_input_tokens?: unknown; - completion_tokens?: unknown; - completion_tokens_details?: { reasoning_tokens?: unknown }; - completion_thinking_tokens?: unknown; - input_tokens_details?: { cached_tokens?: unknown }; - prompt_cache_hit_tokens?: unknown; - prompt_cache_miss_tokens?: unknown; - prompt_cache_write_tokens?: unknown; - prompt_tokens?: unknown; - prompt_tokens_details?: { - cache_creation_tokens?: unknown; - cached_tokens?: unknown; - }; - total_tokens?: unknown; - }; - const outputTokens = toResponsesUsageNumber(value.completion_tokens); - const cachedTokens = toResponsesUsageNumber( - value.prompt_tokens_details?.cached_tokens ?? - value.input_tokens_details?.cached_tokens ?? - value.cache_read_input_tokens ?? - value.prompt_cache_hit_tokens, - ); - const cacheCreationTokens = toResponsesUsageNumber( - value.prompt_tokens_details?.cache_creation_tokens ?? - value.cache_creation_input_tokens ?? - value.prompt_cache_write_tokens, - ); - const reasoningTokens = toResponsesUsageNumber( - value.completion_tokens_details?.reasoning_tokens ?? - value.completion_thinking_tokens, - ); - // Chat usage is the single source of truth for both shapes. Keep the - // Responses counters faithful to it so clients never see zeroed metrics. - // prompt_tokens already covers its cached and created subsets, so the - // split counters are only summed when prompt_tokens is missing. Otherwise - // cached tokens would exceed the reported input total. - const inputTokens = toResponsesUsageNumber( - value.prompt_tokens ?? - toResponsesUsageNumber(value.prompt_cache_miss_tokens) + - cachedTokens + - cacheCreationTokens, - ); - - return { - input_tokens: inputTokens, - input_tokens_details: { cached_tokens: cachedTokens }, - output_tokens: outputTokens, - output_tokens_details: { reasoning_tokens: reasoningTokens }, - total_tokens: - toResponsesUsageNumber(value.total_tokens) || inputTokens + outputTokens, - }; -}; - -const mapChatResponseToResponsesPayload = async ( - accessKeyId: string | null, - credentialFilename: string | null, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - upstreamPayload: Record, - serverToolExecutions: ServerToolExecution[], - imageExecutions: ImageGenerationExecution[] = [], -): Promise> => { - const responseId = createResponseId(); - const choices = Array.isArray(upstreamPayload.choices) - ? upstreamPayload.choices - : []; - const firstChoice = (choices[0] ?? {}) as { - message?: ChatResponseMessage; - }; - const toolCalls = Array.isArray(firstChoice.message?.tool_calls) - ? firstChoice.message.tool_calls - : []; - const outputText = stringifyContent(firstChoice.message?.content); - const createdAt = Math.floor(Date.now() / 1000); - const output: Array> = [ - ...serverToolExecutions.map((execution) => - buildResponsesWebSearchCallItem(execution, 'completed'), - ), - // Image generation is executed locally, so the standard - // `image_generation_call` item has to be synthesized here — the chat - // upstream has no notion of it. - ...imageExecutions.map((execution) => - buildResponsesImageGenerationCallItem(execution), - ), - ]; - const transcriptToolCalls = buildAssistantTranscriptToolCalls( - toolCalls, - defaults.tools, - ); - - // Emit the reasoning as its own item, ahead of the message it produced. - // - // Clients replay `output` verbatim on the next turn, so this is what lets a - // stateless Responses client carry reasoning forward. Without it the only - // reasoning we ever hand back is a transient `reasoning_text.delta`, which no - // client can replay because it has no id and no blob to send back. - // - // `encrypted_content` holds the reasoning verbatim, not ciphertext. Codex - // never opens it — it only echoes it — so plaintext round-trips exactly as - // well, and encrypting would obscure a value that carries no secret: the - // upstream gave us a summary, and the `summary` field below already shows it. - const reasoningText = - firstChoice.message?.reasoning_content ?? - firstChoice.message?.reasoning ?? - ''; - - if (reasoningText) { - output.push({ - id: createResponseReasoningId(), - type: 'reasoning', - summary: [{ type: 'summary_text', text: reasoningText }], - encrypted_content: `${REASONING_PREFIX}${reasoningText}`, - status: 'completed', - }); - } - - if (outputText || !toolCalls.length) { - output.push({ - id: createMessageId(), - type: 'message', - role: 'assistant', - status: 'completed', - content: [ - { - type: 'output_text', - text: outputText, - annotations: [], - }, - ], - }); - } - - toolCalls.forEach((toolCall, index) => { - output.push( - buildResponsesToolCallOutputItem(defaults.tools, { - arguments: toolCall.function?.arguments ?? '', - callId: normalizeToolCallId(toolCall.id, index), - id: createResponseOutputId(), - name: toolCall.function?.name ?? 'function', - status: 'completed', - }), - ); - }); - - await storeResponseSession({ - accessKeyId, - credentialFilename, - createdAt: Date.now(), - id: responseId, - model, - transcript: [ - ...transcript, - { - role: 'assistant', - content: getAssistantTranscriptContent(outputText, transcriptToolCalls), - ...(transcriptToolCalls ? { tool_calls: transcriptToolCalls } : {}), - // A client that continues via `previous_response_id` rather than - // replaying `output` never sees the reasoning item, so the session is - // the only place the reasoning can survive into the next turn. - ...(reasoningText ? { reasoning: reasoningText } : {}), - }, - ], - defaults, - upstreamProtocol: 'chat', - }); - - return { - id: responseId, - object: 'response', - created_at: createdAt, - status: 'completed', - model, - output, - output_text: outputText, - usage: mapChatUsageToResponses(upstreamPayload.usage), - metadata: defaults.metadata ?? {}, - previous_response_id: previousResponseId, - }; -}; - -const buildResponsesWebSearchCallItem = ( - execution: ServerToolExecution | ServerToolInvocation, - status: 'completed' | 'in_progress', - id = `ws_${crypto.randomUUID().replaceAll('-', '')}`, -): Record => ({ - id, - type: 'web_search_call', - status, - action: - execution.type === 'web_search' - ? { type: 'search', query: execution.input.query } - : { - type: 'open_page', - url: - 'result' in execution - ? (execution.result.url ?? execution.input.url) - : execution.input.url, - }, -}); - -/** - * Emits an already-buffered chat payload as a Responses SSE stream. - * - * Used when a request had to be buffered to inspect it — image generation is - * executed locally, so the call cannot be forwarded before it is seen. The - * client still asked for `stream: true`, so the buffered result is replayed as - * the same event sequence a live stream would have produced. - * - * The text is replayed as delta events rather than arriving whole in - * `response.completed`: a client that renders as it reads subscribes to deltas - * and would otherwise show nothing until the turn ends. - */ -const mapChatResponseToResponsesStream = async ( - upstreamPayload: Record, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - proxyContext: ProxyContext, - imageExecutions: ImageGenerationExecution[], - serverToolExecutions: ServerToolExecution[] = [], -): Promise => { - const payload = await mapChatResponseToResponsesPayload( - proxyContext.accessKeyId, - proxyContext.credentialFilename, - defaults, - transcript, - model, - previousResponseId, - upstreamPayload, - serverToolExecutions, - imageExecutions, - ); - // The mapper creates and persists the session id, so the stream has to reuse - // it: advertising a different one would leave a client unable to continue the - // turn, because nothing was stored under the id it was given. - const responseId = String(payload.id); - const output = payload.output as Array>; - const messageIndex = output.findIndex((item) => item.type === 'message'); - const messageItem = - messageIndex === -1 - ? null - : (output[messageIndex] as { - content?: Array<{ text?: string }>; - id?: string; - }); - const messageText = messageItem?.content?.[0]?.text ?? ''; - const otherItems = output - .map((item, output_index) => ({ item, output_index })) - .filter(({ output_index }) => output_index !== messageIndex); - - // The live path announces a server-tool item as in-progress and narrates its - // lifecycle before closing it, and consumers can subscribe to those events. - // A buffered replay that jumps straight to `done` hides the search entirely - // from a client watching for it. - const serverToolFrames = ({ - item, - output_index, - }: { - item: Record; - output_index: number; - }): Array> => { - const itemId = String(item.id ?? ''); - - if (item.type !== 'web_search_call') { - return [ - { - item, - output_index, - response_id: responseId, - type: 'response.output_item.added', - }, - ]; - } - - return [ - { - item: { ...item, status: 'in_progress' }, - output_index, - response_id: responseId, - type: 'response.output_item.added', - }, - { - item_id: itemId, - output_index, - type: 'response.web_search_call.in_progress', - }, - { - item_id: itemId, - output_index, - type: 'response.web_search_call.searching', - }, - { - item_id: itemId, - output_index, - type: 'response.web_search_call.completed', - }, - ]; - }; - - const frames: Array> = [ - { - response: { ...payload, output: [], status: 'in_progress' }, - type: 'response.created', - }, - { - response: { id: responseId, status: 'in_progress' }, - type: 'response.in_progress', - }, - ...otherItems.flatMap(({ item, output_index }) => - serverToolFrames({ item, output_index }), - ), - ...otherItems.map(({ item, output_index }) => ({ - item, - output_index, - response_id: responseId, - type: 'response.output_item.done', - })), - ]; - - // Mirrors the live path: the message item is announced, filled by deltas, - // then closed. No `content_part` events — the live path does not emit them. - if (messageItem && messageIndex !== -1) { - frames.push({ - item: { ...messageItem, status: 'in_progress' }, - output_index: messageIndex, - response_id: responseId, - type: 'response.output_item.added', - }); - - if (messageText) { - frames.push({ - delta: messageText, - item_id: messageItem.id, - output_index: messageIndex, - response_id: responseId, - type: 'response.output_text.delta', - }); - frames.push({ - item: messageItem, - output_index: messageIndex, - response_id: responseId, - text: messageText, - type: 'response.output_text.done', - }); - } - - frames.push({ - item: messageItem, - output_index: messageIndex, - response_id: responseId, - type: 'response.output_item.done', - }); - } - - frames.push({ - response: { ...payload, id: responseId }, - type: 'response.completed', - }); - - const body = [ - ...frames.map((frame) => eventFrameText(String(frame.type), frame)), - DONE_FRAME_TEXT, - '', - ].join('\n\n'); - - return createSseResponse(body); -}; - -const mapChatStreamToResponsesEventStream = ( - upstreamResponse: Response, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - proxyContext: ProxyContext, - responseId = createResponseId(), - providedServerToolItems?: ResponsesServerToolItem[], - emitOpeningEvents = true, - emitServerToolLifecycle = true, - providedOutputIndexAllocator?: () => number, - rejectErrorPayloads = false, -): Response => { - if (!upstreamResponse.ok || !upstreamResponse.body) { - return upstreamResponse; - } - - const serverToolItems = - providedServerToolItems ?? - getServerToolExecutions(upstreamResponse).map((execution, outputIndex) => { - const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; - - return { - completed: buildResponsesWebSearchCallItem(execution, 'completed', id), - inProgress: buildResponsesWebSearchCallItem( - execution, - 'in_progress', - id, - ), - outputIndex, - }; - }); - let outputText = ''; - // Reasoning accumulated from stream deltas. The delta events alone are not - // replayable — a client needs a reasoning item in the completed output, with - // a blob of its own, to send anything back on the next turn. - let streamedReasoning = ''; - // Claimed on the first reasoning delta, which lands before any text, so the - // item sorts ahead of the message it produced. - let reasoningOutputIndex: number | null = null; - let reasoningItemAdded = false; - // Fixed when the first reasoning delta arrives, so the `output_item.added` - // event and the completed output reference the same id. - let reasoningItemId = ''; - - let nextOutputIndex = - serverToolItems.reduce( - (maximum, item) => Math.max(maximum, item.outputIndex), - -1, - ) + 1; - const allocateOutputIndex = - providedOutputIndexAllocator ?? (() => nextOutputIndex++); - const messageState: StreamingMessageState = { - outputIndex: null, - outputItemId: createMessageId(), - }; - let messageAddedEmitted = false; - const toolCallStates = new Map(); - const toolCallStateKeys = new Map(); - let latestUsage: unknown = null; - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - const closer = createStreamCloser(); - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - - const stream = new ReadableStream({ - start: (controller) => { - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - let totalToolArgumentLength = 0; - let streamRejected = false; - - const enqueueEvent = (payload: Record): void => { - const eventType = - typeof payload.type === 'string' ? payload.type : 'message'; - controller.enqueue( - encoder.encode( - `event: ${eventType}\ndata: ${JSON.stringify(payload)}\n\n`, - ), - ); - }; - - const buildStreamingMessageItem = ( - status: 'completed' | 'in_progress', - ): Record => ({ - id: messageState.outputItemId, - type: 'message', - role: 'assistant', - status, - content: [ - { - type: 'output_text', - text: outputText, - annotations: [], - }, - ], - }); - - // Carries the reasoning verbatim rather than encrypted — same reasoning as - // the non-streaming item above. - const buildStreamingReasoningItem = (): Record => ({ - id: reasoningItemId, - type: 'reasoning', - summary: [{ type: 'summary_text', text: streamedReasoning }], - encrypted_content: `${REASONING_PREFIX}${streamedReasoning}`, - status: 'completed', - }); - - const ensureReasoningItemAdded = (): void => { - if (reasoningItemAdded) { - return; - } - - reasoningOutputIndex ??= allocateOutputIndex(); - enqueueEvent({ - type: 'response.output_item.added', - item: buildStreamingReasoningItem(), - output_index: reasoningOutputIndex, - response_id: responseId, - }); - reasoningItemAdded = true; - }; - - const ensureMessageAdded = (): void => { - if (messageAddedEmitted) { - return; - } - - messageState.outputIndex ??= allocateOutputIndex(); - enqueueEvent({ - type: 'response.output_item.added', - item: buildStreamingMessageItem('in_progress'), - output_index: messageState.outputIndex, - response_id: responseId, - }); - messageAddedEmitted = true; - }; - - if (emitOpeningEvents) { - enqueueEvent({ - type: 'response.created', - response: { - id: responseId, - object: 'response', - created_at: Math.floor(Date.now() / 1000), - model, - output: [], - }, - }); - enqueueEvent({ - type: 'response.in_progress', - response: { - id: responseId, - status: 'in_progress', - }, - }); - } - if (emitServerToolLifecycle) { - serverToolItems.forEach(({ completed, inProgress, outputIndex }) => { - const itemId = String(inProgress.id); - enqueueEvent({ - type: 'response.output_item.added', - item: inProgress, - output_index: outputIndex, - response_id: responseId, - }); - enqueueEvent({ - type: 'response.web_search_call.in_progress', - item_id: itemId, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.web_search_call.searching', - item_id: itemId, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.web_search_call.completed', - item_id: itemId, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.output_item.done', - item: completed, - output_index: outputIndex, - response_id: responseId, - }); - }); - } - - const maybeEmitToolCallAdded = ( - toolCallState: StreamingToolCallState, - allowIncompleteName = false, - ): void => { - if (toolCallState.addedEmitted) { - return; - } - - const shouldWaitForInitialName = - !allowIncompleteName && - Boolean(defaults.tools?.length) && - toolCallState.name.length === 0; - const shouldWaitForMoreName = - !allowIncompleteName && - defaults.tools?.length && - toolCallState.name.length > 0 && - hasSupportedLongerToolNamePrefix(defaults.tools, toolCallState.name); - - if (shouldWaitForInitialName || shouldWaitForMoreName) { - return; - } - - enqueueEvent({ - type: 'response.output_item.added', - item: buildResponsesToolCallOutputItem(defaults.tools, { - arguments: '', - callId: toolCallState.callId, - id: toolCallState.outputItemId, - name: toolCallState.name || 'function', - status: 'in_progress', - }), - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - - toolCallState.addedEmitted = true; - toolCallState.pendingArgumentDeltas.forEach((delta) => { - enqueueEvent({ - type: getResponsesToolCallArgumentDeltaEventType( - defaults.tools, - toolCallState.name, - ), - delta, - item_id: toolCallState.outputItemId, - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - }); - toolCallState.pendingArgumentDeltas = []; - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - const transcriptToolCalls = - buildStreamingAssistantTranscriptToolCalls( - [...toolCallStates.values()], - defaults.tools, - ); - try { - await storeResponseSession({ - accessKeyId: proxyContext.accessKeyId, - credentialFilename: proxyContext.credentialFilename, - createdAt: Date.now(), - id: responseId, - model, - transcript: [ - ...transcript, - { - role: 'assistant', - content: getAssistantTranscriptContent( - outputText, - transcriptToolCalls, - ), - ...(transcriptToolCalls - ? { tool_calls: transcriptToolCalls } - : {}), - // Same reason as the non-streaming path: a client that - // continues via `previous_response_id` instead of - // replaying `output` would otherwise lose the reasoning. - ...(streamedReasoning - ? { reasoning: streamedReasoning } - : {}), - }, - ], - defaults, - upstreamProtocol: 'chat', - }); - } catch (error) { - console.error( - '[CodeBuddy2API] Failed to persist Responses session', - error, - ); - enqueueEvent({ - type: 'response.error', - error: { message: 'Failed to persist response session' }, - }); - controller.enqueue(encodeDoneFrame()); - releaseReader(); - controller.close(); - return; - } - [...toolCallStates.values()].forEach((toolCallState) => { - maybeEmitToolCallAdded(toolCallState, true); - enqueueEvent({ - type: 'response.output_item.done', - item: buildResponsesToolCallOutputItem(defaults.tools, { - arguments: toolCallState.arguments, - callId: toolCallState.callId, - id: toolCallState.outputItemId, - name: toolCallState.name || 'function', - status: 'completed', - }), - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - enqueueEvent({ - type: getResponsesToolCallArgumentDeltaEventType( - defaults.tools, - toolCallState.name, - ).replace('.delta', '.done'), - arguments: toolCallState.arguments, - item_id: toolCallState.outputItemId, - output_index: toolCallState.outputIndex, - response_id: responseId, - }); - }); - if (outputText) { - ensureMessageAdded(); - enqueueEvent({ - type: 'response.output_text.done', - item: buildStreamingMessageItem('completed'), - output_index: messageState.outputIndex, - response_id: responseId, - text: outputText, - }); - enqueueEvent({ - type: 'response.output_item.done', - item: buildStreamingMessageItem('completed'), - output_index: messageState.outputIndex, - response_id: responseId, - }); - } - enqueueEvent({ - type: 'response.completed', - response: { - id: responseId, - status: 'completed', - output_text: outputText, - previous_response_id: previousResponseId, - usage: mapChatUsageToResponses(latestUsage), - output: [ - ...serverToolItems.map(({ completed, outputIndex }) => ({ - item: completed, - outputIndex, - })), - // Streamed reasoning needs the same replayable item the - // non-streaming path emits. It sorts ahead of the message by - // taking the next index before the message claims its own — - // the deltas come first on the wire, so the item order has - // to match or a client replaying `output` scrambles it. - ...(streamedReasoning && reasoningOutputIndex !== null - ? [ - { - item: buildStreamingReasoningItem(), - outputIndex: reasoningOutputIndex, - }, - ] - : []), - ...(outputText && messageState.outputIndex !== null - ? [ - { - item: buildStreamingMessageItem('completed'), - outputIndex: messageState.outputIndex, - }, - ] - : []), - ...[...toolCallStates.values()].map((toolCallState) => ({ - item: buildResponsesToolCallOutputItem(defaults.tools, { - arguments: toolCallState.arguments, - callId: toolCallState.callId, - id: toolCallState.outputItemId, - name: toolCallState.name || 'function', - status: 'completed', - }), - outputIndex: toolCallState.outputIndex, - })), - ] - .sort((left, right) => left.outputIndex - right.outputIndex) - .map(({ item }) => item), - }, - }); - controller.enqueue(encodeDoneFrame()); - releaseReader(); - controller.close(); - return; - } - - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_BUFFER_LENGTH) { - buffer = ''; - } - - for (const frame of frames) { - if (streamRejected) { - break; - } - if (frame.length > MAX_STREAM_BUFFER_LENGTH) { - continue; - } - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - continue; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - // The upstream here is the chat pipeline, which reports a deadline - // as a terminal error chunk and closes cleanly. Surfacing it keeps - // the client from seeing an empty successful response. - const upstreamError = readTimeoutFrame(frame); - - if (upstreamError !== null) { - streamRejected = true; - enqueueEvent({ - type: 'response.error', - error: { message: upstreamError }, - }); - break; - } - - try { - const payload = JSON.parse(raw) as { - choices?: Array<{ - delta?: { - content?: string; - reasoning_content?: string; - tool_calls?: ChatResponseToolCall[]; - }; - }>; - error?: unknown; - usage?: unknown; - }; - if (rejectErrorPayloads && payload.error) { - const error = - typeof payload.error === 'object' - ? (payload.error as { message?: unknown }) - : null; - streamRejected = true; - enqueueEvent({ - type: 'response.error', - error: { - message: - typeof error?.message === 'string' - ? error.message - : typeof payload.error === 'string' - ? payload.error - : 'Upstream request failed', - }, - }); - break; - } - // The final upstream chunk carries the aggregated usage, so - // remember it for the downstream response.completed event. - if (payload.usage !== undefined) { - latestUsage = payload.usage; - } - const delta = payload.choices?.[0]?.delta; - - if (delta?.content) { - ensureMessageAdded(); - outputText = `${outputText}${delta.content}`; - if (outputText.length > MAX_STREAM_TEXT_LENGTH) { - throw new Error('Response output exceeds the maximum size'); - } - enqueueEvent({ - type: 'response.output_text.delta', - delta: delta.content, - // Send only the item reference: embedding the accumulated - // text re-serializes it on every delta, which makes the - // enqueued volume quadratic in the output size. The full - // text still arrives intact in response.output_text.done - // and response.completed. - item_id: messageState.outputItemId, - output_index: messageState.outputIndex, - response_id: responseId, - }); - } - - if (delta?.reasoning_content) { - reasoningItemId ||= createResponseReasoningId(); - streamedReasoning += delta.reasoning_content; - ensureReasoningItemAdded(); - enqueueEvent({ - type: 'response.reasoning_text.delta', - delta: delta.reasoning_content, - response_id: responseId, - }); - } - - delta?.tool_calls?.forEach((toolCall, position) => { - const lookupKeys = getStreamingToolCallLookupKeys( - toolCall, - position, - ); - const existingCanonicalKey = lookupKeys - .map((key) => toolCallStateKeys.get(key) ?? key) - .find((key) => toolCallStates.has(key)); - const canonicalKey = - existingCanonicalKey ?? - getStreamingToolCallCanonicalKey(toolCall, position); - const existing = toolCallStates.get(canonicalKey); - const outputIndex = existing - ? existing.outputIndex - : allocateOutputIndex(); - const current = existing ?? { - addedEmitted: false, - arguments: '', - canonicalKey, - callId: normalizeToolCallId(toolCall.id, outputIndex), - name: '', - outputIndex, - outputItemId: createResponseOutputId(), - pendingArgumentDeltas: [], - }; - - if (toolCall.function?.name) { - if ( - current.name.length + toolCall.function.name.length > - MAX_TOOL_NAME_LENGTH - ) { - throw new Error( - 'Response tool name exceeds the maximum size', - ); - } - current.name += toolCall.function.name; - } - maybeEmitToolCallAdded(current); - - if (toolCall.function?.arguments) { - if ( - current.arguments.length + - toolCall.function.arguments.length > - MAX_TOOL_ARGUMENT_LENGTH - ) { - throw new Error( - 'Response tool arguments exceed the maximum size', - ); - } - if ( - totalToolArgumentLength + - toolCall.function.arguments.length > - MAX_RESPONSE_SESSION_TOTAL_BYTES - ) { - throw new Error( - 'Response tool arguments exceed the maximum size', - ); - } - totalToolArgumentLength += toolCall.function.arguments.length; - current.arguments = `${current.arguments}${toolCall.function.arguments}`; - if (current.addedEmitted) { - enqueueEvent({ - type: getResponsesToolCallArgumentDeltaEventType( - defaults.tools, - current.name, - ), - delta: toolCall.function.arguments, - item_id: current.outputItemId, - output_index: current.outputIndex, - response_id: responseId, - }); - } else { - current.pendingArgumentDeltas.push( - toolCall.function.arguments, - ); - } - } - - toolCallStates.set(canonicalKey, current); - lookupKeys.forEach((key) => { - toolCallStateKeys.set(key, current.canonicalKey); - }); - }); - } catch (error) { - if ( - error instanceof Error && - error.message.includes('maximum size') - ) { - streamRejected = true; - } - console.error( - '[CodeBuddy2API] Failed to parse upstream SSE frame', - { - route: '/v1/responses', - frame: raw.slice(0, 1000), - }, - ); - enqueueEvent({ - type: 'response.error', - error: { - message: 'Failed to parse upstream SSE frame', - }, - }); - } - } - - if (streamRejected) { - try { - await reader!.cancel(); - } finally { - releaseReader(); - controller.close(); - } - return; - } - } - }; - - void pump().catch((error) => { - if (cancelled) return; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage === null) { - closer.mark(); - controller.error(error); - return; - } - - streamRejected = true; - void reader?.cancel().then( - () => undefined, - () => undefined, - ); - releaseReader(); - closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - releaseReader(); - } - }, - }); - - return createSseResponse(stream, { status: 200 }); -}; - -const createResponsesEventStream = async ( - request: NextRequest, - defaults: ResponseSessionDefaults, - transcript: TranscriptMessage[], - model: string, - previousResponseId: string | null, - maxOutputTokens: number | undefined, - proxyContext: ProxyContext, - debugTrace?: DebugTrace, -): Promise => { - const translatedTools = translateResponsesToolsToChat(defaults.tools); - const translatedToolNames = new Set( - ( - (translatedTools ?? []) as Array<{ - function: { name: string }; - }> - ).map((tool) => normalizeToolName(tool.function.name)), - ); - const [searchEnabled, fetchEnabled] = await Promise.all([ - translatedToolNames.has(normalizeToolName(WEB_SEARCH_TOOL_NAME)) - ? isWebSearchEnabled() - : false, - translatedToolNames.has(normalizeToolName(WEB_FETCH_TOOL_NAME)) - ? isWebFetchEnabled() - : false, - ]); - - const chatBody = { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames(transcript, defaults.tools), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }; - - // Image generation is executed locally, so a streaming request has to be - // buffered first to see whether the model asked for an image. Without this - // the call is forwarded as an ordinary function_call the client is expected - // to resolve — and nothing would ever generate the image. - // - // Handled before the server-tool branch below: a turn may declare both, and - // gating on search/fetch would silently skip generation whenever those were - // enabled. - if (hasImageGenerationTool(defaults.tools)) { - const { executions, response, serverToolExecutions } = - await executeImageGenerationLoop({ - body: chatBody, - // Buffered so the tool call can be inspected before any delta reaches - // the client; the ordinary path below stays live. - callUpstream: (loopBody) => - proxyChatCompletions( - request, - { ...loopBody, stream: false } as never, - proxyContext, - debugTrace, - '/v1/responses', - ), - context: proxyContext, - request, - }); - - // Always consumed, even when nothing was generated: the loop has already - // sent the turn upstream, and re-issuing it would bill twice and could - // return a different answer than the one inspected. - if (!response.ok) { - return response; - } - - return mapChatResponseToResponsesStream( - (await response.json()) as Record, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - executions, - serverToolExecutions, - ); - } - - if (!searchEnabled && !fetchEnabled) { - const upstreamResponse = await proxyChatCompletions( - request, - chatBody as never, - proxyContext, - debugTrace, - '/v1/responses', - ); - - return mapChatStreamToResponsesEventStream( - upstreamResponse, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - ); - } - - const encoder = new TextEncoder(); - const responseId = createResponseId(); - const serverToolItems: ResponsesServerToolItem[] = []; - const itemsByInvocationId = new Map(); - let nextOutputIndex = 0; - const allocateOutputIndex = (): number => nextOutputIndex++; - let activeReader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - - const stream = new ReadableStream({ - start: (controller) => { - const enqueueEvent = ( - payload: Record & { type: string }, - ): void => { - if (cancelled) return; - controller.enqueue( - encoder.encode( - `event: ${payload.type}\ndata: ${JSON.stringify(payload)}\n\n`, - ), - ); - }; - - enqueueEvent({ - type: 'response.created', - response: { - id: responseId, - object: 'response', - created_at: Math.floor(Date.now() / 1000), - model, - output: [], - }, - }); - enqueueEvent({ - type: 'response.in_progress', - response: { id: responseId, status: 'in_progress' }, - }); - - const run = async (): Promise => { - const upstreamResponse = await proxyChatCompletions( - request, - { - model, - messages: [ - ...(defaults.instructions - ? [{ role: 'system', content: defaults.instructions }] - : []), - ...normalizeTranscriptMessageToolNames( - transcript, - defaults.tools, - ), - ], - max_tokens: maxOutputTokens, - stream: true, - tools: translatedTools, - tool_choice: translateResponsesToolChoiceToChatWithTools( - defaults.tools, - defaults.tool_choice, - ), - }, - proxyContext, - debugTrace, - '/v1/responses', - { - emitStreamEvents: true, - onCall: (invocation) => { - const outputIndex = allocateOutputIndex(); - const id = `ws_${crypto.randomUUID().replaceAll('-', '')}`; - const item = { - completed: buildResponsesWebSearchCallItem( - invocation, - 'completed', - id, - ), - inProgress: buildResponsesWebSearchCallItem( - invocation, - 'in_progress', - id, - ), - outputIndex, - }; - serverToolItems.push(item); - itemsByInvocationId.set(invocation.id, item); - enqueueEvent({ - type: 'response.output_item.added', - item: item.inProgress, - output_index: outputIndex, - response_id: responseId, - }); - enqueueEvent({ - type: 'response.web_search_call.in_progress', - item_id: id, - output_index: outputIndex, - }); - enqueueEvent({ - type: 'response.web_search_call.searching', - item_id: id, - output_index: outputIndex, - }); - }, - onResult: (execution) => { - const item = itemsByInvocationId.get(execution.id)!; - const id = String(item.inProgress.id); - item.completed = buildResponsesWebSearchCallItem( - execution, - 'completed', - id, - ); - enqueueEvent({ - type: 'response.web_search_call.completed', - item_id: id, - output_index: item.outputIndex, - }); - enqueueEvent({ - type: 'response.output_item.done', - item: item.completed, - output_index: item.outputIndex, - response_id: responseId, - }); - }, - }, - ); - - if (cancelled) { - await upstreamResponse.body?.cancel(); - return; - } - - if (!upstreamResponse.ok || !upstreamResponse.body) { - enqueueEvent({ - type: 'response.error', - error: { message: 'Upstream request failed' }, - }); - controller.enqueue(encodeDoneFrame()); - controller.close(); - return; - } - - const mappedResponse = mapChatStreamToResponsesEventStream( - upstreamResponse, - defaults, - transcript, - model, - previousResponseId, - proxyContext, - responseId, - serverToolItems, - false, - false, - allocateOutputIndex, - true, - ); - const reader = mappedResponse.body!.getReader(); - activeReader = reader; - - while (true) { - const { done, value } = await reader.read(); - if (cancelled) return; - if (done) break; - controller.enqueue(value); - } - - reader.releaseLock(); - activeReader = null; - controller.close(); - }; +// --------------------------------------------------------------------------- +// /v1/responses route entry point +// +// The machinery this route relies on lives in the sibling modules: +// +// responses-types shared type declarations +// responses-session session persistence +// responses-tools Responses <-> Chat tool translation +// responses-transcript transcript construction and usage mapping +// responses-payload non-streaming payload assembly +// responses-stream SSE stream mapping +// --------------------------------------------------------------------------- - void run().catch((error) => { - if (!cancelled) controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - await activeReader?.cancel(reason); - activeReader?.releaseLock(); - activeReader = null; - }, - }); +import type { NextRequest } from 'next/server'; - return createSseResponse(stream); -}; +import { getDefaultModel } from '../domain/config'; +import { getCredentialSupportedModels } from '../domain/credentials'; +import type { DebugTrace } from '../domain/debug'; +import { createErrorResponse } from '../shared/http'; +import { resolveRequestAccessKey } from './auth'; +import { + proxyChatCompletions, + proxyResponsesUpstream, + resolveProxyContext, + resolveProxyContextByCredentialFilename, +} from './codebuddy'; +import { executeImageGenerationLoop } from './image-generation'; +import { mapChatResponseToResponsesPayload } from './responses-payload'; +import { + getResponseSession, + getValidatedPreviousSession, + storeUpstreamResponseBinding, +} from './responses-session'; +import { createResponsesEventStream } from './responses-event-stream'; +import { + getResponsesCompatibilityError, + hasImageGenerationTool, + normalizeTranscriptMessageToolNames, + translateResponsesToolsToChat, + translateResponsesToolChoiceToChatWithTools, +} from './responses-tools'; +import { prepareTranscript } from './responses-transcript'; +import type { ResponsesRequestBody } from './responses-types'; +import { getServerToolExecutions } from './web-search-loop'; export const handleResponsesRequest = async ( request: NextRequest, @@ -2954,8 +283,5 @@ export const handleResponsesRequest = async ( } }; -export const resetResponseSessions = (): void => { - getSessionStore().clear(); - getSessionByteStore().clear(); - setSessionTotalBytes(0); -}; +export { translateResponsesToolsToChat } from './responses-tools'; +export { resetResponseSessions } from './responses-session'; From 423abca7f5e9b3e09d4cb5f41746987de94d9ae9 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 22:00:43 +0800 Subject: [PATCH 5/7] refactor(codebuddy): split the chat and Responses proxy into focused modules codebuddy.ts was 3493 lines and the largest file in the repo. Wire types, credential-context resolution, usage tracking, upstream request building, the Responses translations, chat stream handling, server-tool detection and model discovery now each have their own module. The original keeps the three entry points plus the re-export block, so no importer changes. --- lib/server/proxy/codebuddy-chat-stream.ts | 474 +++ lib/server/proxy/codebuddy-context.ts | 159 + lib/server/proxy/codebuddy-models.ts | 242 ++ .../proxy/codebuddy-responses-request.ts | 382 +++ .../proxy/codebuddy-responses-response.ts | 700 ++++ lib/server/proxy/codebuddy-server-tools.ts | 239 ++ lib/server/proxy/codebuddy-types.ts | 137 + lib/server/proxy/codebuddy-upstream.ts | 272 ++ lib/server/proxy/codebuddy-usage-stream.ts | 192 ++ lib/server/proxy/codebuddy-usage.ts | 152 + lib/server/proxy/codebuddy.ts | 2971 +---------------- 11 files changed, 3033 insertions(+), 2887 deletions(-) create mode 100644 lib/server/proxy/codebuddy-chat-stream.ts create mode 100644 lib/server/proxy/codebuddy-context.ts create mode 100644 lib/server/proxy/codebuddy-models.ts create mode 100644 lib/server/proxy/codebuddy-responses-request.ts create mode 100644 lib/server/proxy/codebuddy-responses-response.ts create mode 100644 lib/server/proxy/codebuddy-server-tools.ts create mode 100644 lib/server/proxy/codebuddy-types.ts create mode 100644 lib/server/proxy/codebuddy-upstream.ts create mode 100644 lib/server/proxy/codebuddy-usage-stream.ts create mode 100644 lib/server/proxy/codebuddy-usage.ts diff --git a/lib/server/proxy/codebuddy-chat-stream.ts b/lib/server/proxy/codebuddy-chat-stream.ts new file mode 100644 index 0000000..5822e53 --- /dev/null +++ b/lib/server/proxy/codebuddy-chat-stream.ts @@ -0,0 +1,474 @@ +import { createErrorResponse } from '../shared/http'; +import { createSseResponse } from '../shared/sse'; +import { + chatStreamErrorChunks, + createStreamCloser, + toUpstreamTimeoutMessage, +} from '../shared/upstream-timeout'; +import { recordProxyUsage } from './codebuddy-usage'; +import { + type ChatStreamChunk, + type ChatStreamDelta, + CORS_HEADERS, + MAX_STREAM_FRAME_LENGTH, + type ProxyContext, + type ToolCallChunk, + type ToolCallMapping, + type ToolCallNormalizationState, +} from './codebuddy-types'; + +export const aggregateToolCalls = ( + toolCalls: NonNullable, +): Array<{ + id?: string; + type?: string; + function: { + arguments: string; + name: string; + }; +}> => { + const aggregated = new Map< + string, + { + order: number; + id?: string; + type?: string; + function: { + arguments: string; + name: string; + }; + } + >(); + const latestKeyByIndex = new Map(); + + toolCalls.forEach((toolCall, position) => { + const normalizedId = createNormalizedToolCallId(toolCall.id, position); + const key = + (toolCall.id ? `id:${normalizedId}` : undefined) ?? + (typeof toolCall.index === 'number' + ? latestKeyByIndex.get(toolCall.index) + : undefined) ?? + `position:${position}`; + const current = aggregated.get(key) ?? { + order: aggregated.size, + function: { + arguments: '', + name: '', + }, + }; + + if (toolCall.id) { + current.id = normalizedId; + } + + if (toolCall.type) { + current.type = toolCall.type; + } + + if (toolCall.function?.name) { + current.function.name += toolCall.function.name; + } + + if (toolCall.function?.arguments) { + current.function.arguments += toolCall.function.arguments; + } + + aggregated.set(key, current); + + if (typeof toolCall.index === 'number') { + latestKeyByIndex.set(toolCall.index, key); + } + }); + + return [...aggregated.values()] + .sort((left, right) => left.order - right.order) + .map(({ order: _order, ...value }, index) => ({ + ...value, + id: value.id ?? createNormalizedToolCallId(undefined, index), + })); +}; + +export const getToolCallStateKey = ( + toolCall: ToolCallChunk, + position: number, +): string => { + if (toolCall.id) { + return `id:${toolCall.id}`; + } + + if (typeof toolCall.index === 'number') { + return `index:${toolCall.index}`; + } + + return `position:${position}`; +}; + +export const createNormalizedToolCallId = ( + sourceId: string | undefined, + normalizedIndex: number, +): string => { + if (sourceId && !sourceId.startsWith('tooluse_')) { + return sourceId; + } + + const suffix = + sourceId?.replace(/^tooluse_/, '') ?? + `${normalizedIndex}_${crypto.randomUUID().replaceAll('-', '')}`; + + return `call_${suffix}`; +}; + +export const resolveToolCallMapping = ( + state: ToolCallNormalizationState, + toolCall: ToolCallChunk, + position: number, +): ToolCallMapping => { + const keys = toolCall.id + ? [`id:${toolCall.id}`] + : [ + typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, + `position:${position}`, + ].filter((value): value is string => value !== null); + const existing = keys + .map((key) => state.mappings.get(key)) + .find((value) => value !== undefined); + + if (existing) { + return existing; + } + + return { + id: createNormalizedToolCallId(toolCall.id, state.nextIndex), + index: state.nextIndex++, + }; +}; + +export const normalizeStreamToolCalls = ( + chunk: ChatStreamChunk, + state: ToolCallNormalizationState, +): ChatStreamChunk => { + if (!chunk.choices?.length) { + return chunk; + } + + return { + ...chunk, + choices: chunk.choices.map((choice) => { + if (!choice.delta?.tool_calls?.length) { + return choice; + } + + return { + ...choice, + delta: { + ...choice.delta, + tool_calls: choice.delta.tool_calls.map((toolCall, position) => { + const mapping = resolveToolCallMapping(state, toolCall, position); + const sourceKey = getToolCallStateKey(toolCall, position); + + state.mappings.set(sourceKey, mapping); + + if (toolCall.id) { + state.mappings.set(`id:${toolCall.id}`, mapping); + } + + if (typeof toolCall.index === 'number') { + state.mappings.set(`index:${toolCall.index}`, mapping); + } + + return { + ...toolCall, + id: mapping.id, + index: mapping.index, + }; + }), + }, + }; + }), + }; +}; + +export const normalizeStreamingResponse = ({ + model, + proxyContext, + route, + upstreamResponse, +}: { + model: string; + proxyContext: ProxyContext; + route: string; + upstreamResponse: Response; +}): Response => { + if (!upstreamResponse.body) { + return createSseResponse(null, { + headers: CORS_HEADERS, + status: upstreamResponse.status, + }); + } + + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const state: ToolCallNormalizationState = { + mappings: new Map(), + nextIndex: 0, + }; + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + const closer = createStreamCloser(); + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + + const stream = new ReadableStream({ + start: (controller) => { + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + let latestUsage: unknown = null; + + const processFrame = (frame: string): string => { + const lines = frame.split('\n'); + const lineIndex = lines.findIndex((line) => line.startsWith('data: ')); + + if (lineIndex === -1) { + return frame; + } + + const raw = lines[lineIndex]?.slice(6).trim() ?? ''; + + if (!raw || raw === '[DONE]') { + return frame; + } + + try { + const chunk = JSON.parse(raw) as ChatStreamChunk; + if (chunk.usage !== undefined) { + latestUsage = chunk.usage; + } + const normalized = normalizeStreamToolCalls(chunk, state); + lines[lineIndex] = `data: ${JSON.stringify(normalized)}`; + return lines.join('\n'); + } catch { + return frame; + } + }; + + const flushFrames = (frames: string[]): void => { + frames.forEach((frame) => { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + return; + } + controller.enqueue(encoder.encode(`${processFrame(frame)}\n\n`)); + }); + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + if (buffer.trim()) { + flushFrames([buffer]); + } + + await recordProxyUsage({ + model, + proxyContext, + route, + usage: latestUsage, + }); + + releaseReader(); + try { + controller.close(); + } catch { + // The downstream stream may have been cancelled while the pump was completing. + } + return; + } + + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + try { + await reader!.cancel(); + } finally { + releaseReader(); + controller.close(); + } + return; + } + flushFrames(frames); + } + }; + + void pump().catch((error) => { + if (cancelled) return; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage !== null) { + closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); + return; + } + + controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + releaseReader(); + } + }, + }); + + return createSseResponse(stream, { + headers: CORS_HEADERS, + status: upstreamResponse.status, + }); +}; + +export const aggregateUpstreamStream = async ( + upstreamResponse: Response, + fallbackModel: string, +): Promise<{ model: string; response: Response; usage: unknown }> => { + const payloadText = await upstreamResponse.text(); + const toolCalls: NonNullable = []; + let responseId = ''; + let responseObject = 'chat.completion'; + let created = Math.floor(Date.now() / 1000); + let model = fallbackModel; + let content = ''; + let reasoningContent = ''; + let finishReason: string | null = 'stop'; + let role = 'assistant'; + let usage: unknown = null; + + for (const frame of payloadText.split('\n\n')) { + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + continue; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + let chunk: ChatStreamChunk; + + try { + chunk = JSON.parse(raw) as ChatStreamChunk; + } catch { + return { + model: fallbackModel, + response: createErrorResponse( + 502, + 'Failed to parse upstream SSE frame', + ), + usage: null, + }; + } + + if (chunk.id) { + responseId = chunk.id; + } + + if (chunk.object) { + responseObject = chunk.object.replace(/\.chunk$/, ''); + } + + if (typeof chunk.created === 'number') { + created = chunk.created; + } + + if (chunk.model) { + model = chunk.model; + } + + if (chunk.usage !== undefined) { + usage = chunk.usage; + } + + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + + if (delta?.role) { + role = delta.role; + } + + if (delta?.content) { + content += delta.content; + } + + if (delta?.reasoning_content ?? delta?.reasoning) { + reasoningContent += delta.reasoning_content ?? delta.reasoning; + } + + if (delta?.tool_calls?.length) { + toolCalls.push(...delta.tool_calls); + } + + if (choice?.finish_reason !== undefined) { + finishReason = choice.finish_reason ?? finishReason; + } + } + + const aggregatedToolCalls = aggregateToolCalls(toolCalls); + const message: Record = { + role, + content: content || null, + }; + + if (reasoningContent) { + message.reasoning_content = reasoningContent; + } + + if (aggregatedToolCalls.length) { + message.tool_calls = aggregatedToolCalls; + } + + return { + model, + response: Response.json({ + id: responseId || `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`, + object: responseObject, + created, + model, + choices: [ + { + index: 0, + message, + finish_reason: + finishReason ?? + (aggregatedToolCalls.length ? 'tool_calls' : 'stop'), + }, + ], + usage, + }), + usage, + }; +}; diff --git a/lib/server/proxy/codebuddy-context.ts b/lib/server/proxy/codebuddy-context.ts new file mode 100644 index 0000000..5de5ea3 --- /dev/null +++ b/lib/server/proxy/codebuddy-context.ts @@ -0,0 +1,159 @@ +import type { NextRequest } from 'next/server'; + +import { resolveRequestAccessKey } from './auth'; +import { + type CredentialRecord, + findEligibleCredentialRecordByFilename, + findCredentialRecordByFilename, + getCredentialProxySettings, + resolveCredentialForRequest, +} from '../domain/credentials'; +import { getRequestHeaderMap } from '../shared/http'; +import type { ProxyContext } from './codebuddy-types'; + +export const getCredentialAffinityKey = ( + request: NextRequest, + accessKeyId: string | null, +): string | undefined => { + const incoming = getRequestHeaderMap(request.headers); + const conversationId = incoming['x-conversation-id']?.trim(); + + if (!conversationId) { + return undefined; + } + + if (accessKeyId) { + return `access-key:${accessKeyId}:conversation:${conversationId}`; + } + + return `global:conversation:${conversationId}`; +}; + +export const getCredentialValue = ( + value: unknown, + candidateKeys: string[], +): string | number | null => { + if (Array.isArray(value)) { + for (const item of value) { + const nested = getCredentialValue(item, candidateKeys); + + if (nested !== null && nested !== '') { + return nested; + } + } + + return null; + } + + if (value && typeof value === 'object') { + for (const key of candidateKeys) { + const direct = (value as Record)[key]; + + if (direct !== undefined && direct !== null && direct !== '') { + return direct as string | number; + } + } + + for (const nestedValue of Object.values(value as Record)) { + const nested = getCredentialValue(nestedValue, candidateKeys); + + if (nested !== null && nested !== '') { + return nested; + } + } + } + + return null; +}; + +export const resolveProxyContext = async ( + request: NextRequest, + model?: string, +): Promise => { + const accessKey = await resolveRequestAccessKey(request); + const credential = await resolveCredentialForRequest({ + accessKeyId: accessKey?.id, + affinityKey: getCredentialAffinityKey(request, accessKey?.id ?? null), + allowedCredentialFilenames: accessKey?.credentialFilenames, + model, + }); + + if (!credential) { + throw new Error('No valid CodeBuddy credentials found'); + } + + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + if (!bearerToken) { + throw new Error('Saved credential does not include a bearer token'); + } + + return { + accessKeyId: accessKey?.id ?? null, + accessKeyName: accessKey?.name ?? null, + auth: { + type: 'bearer', + bearerToken, + userId: String(credential.data.user_id ?? 'unknown'), + credentialData: credential.data, + }, + credentialFilename: credential.filename, + preferences: getCredentialProxySettings(credential.data), + }; +}; + +export const createProxyContextFromCredential = ( + credential: CredentialRecord, +): ProxyContext => { + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + if (!bearerToken) { + throw new Error('Saved credential does not include a bearer token'); + } + + return { + accessKeyId: null, + accessKeyName: null, + auth: { + type: 'bearer', + bearerToken, + userId: String(credential.data.user_id ?? 'unknown'), + credentialData: credential.data, + }, + credentialFilename: credential.filename, + preferences: getCredentialProxySettings(credential.data), + }; +}; + +export const resolveProxyContextByCredentialFilename = async ( + filename: string, + options?: { + accessKey?: { + id?: string | null; + name?: string | null; + }; + allowedCredentialFilenames?: string[]; + requireEligible?: boolean; + }, +): Promise => { + const credential = options?.requireEligible + ? await findEligibleCredentialRecordByFilename( + filename, + options.allowedCredentialFilenames, + ) + : await findCredentialRecordByFilename(filename); + + if (!credential) { + throw new Error('Selected credential was not found'); + } + + return { + ...createProxyContextFromCredential(credential), + accessKeyId: options?.accessKey?.id ?? null, + accessKeyName: options?.accessKey?.name ?? null, + }; +}; diff --git a/lib/server/proxy/codebuddy-models.ts b/lib/server/proxy/codebuddy-models.ts new file mode 100644 index 0000000..0c9a285 --- /dev/null +++ b/lib/server/proxy/codebuddy-models.ts @@ -0,0 +1,242 @@ +import type { NextRequest } from 'next/server'; + +import { resolveRequestAccessKey } from './auth'; +import { getCodeBuddyApiEndpoint } from '../domain/config'; +import { + type CredentialData, + type CredentialRecord, + getCredentialSupportedModels, + listEligibleCredentialRecords, +} from '../domain/credentials'; +import { getCredentialValue } from './codebuddy-context'; +import type { DiscoveredModel } from './codebuddy-types'; + +export const getModelsForCredential = async ({ + bearerToken, + credentialData, +}: { + bearerToken: string; + credentialData: CredentialData; +}): Promise => { + const configuredEndpoint = await getCodeBuddyApiEndpoint(); + const headers = new Headers({ + Accept: 'application/json', + Authorization: `Bearer ${bearerToken}`, + 'X-Product': 'SaaS', + }); + const domain = getCredentialValue(credentialData, ['domain']); + const apiEndpoint = String(domain ?? '') + .toLowerCase() + .endsWith('workbuddy.ai') + ? 'https://www.workbuddy.ai' + : configuredEndpoint; + const enterpriseId = getCredentialValue(credentialData, [ + 'enterprise_id', + 'enterpriseId', + ]); + const tenantId = + getCredentialValue(credentialData, ['tenant_id', 'tenantId']) ?? + enterpriseId; + const userId = getCredentialValue(credentialData, ['user_id', 'userId']); + + if (domain) { + headers.set('X-Domain', String(domain)); + } + + if (enterpriseId) { + headers.set('X-Enterprise-Id', String(enterpriseId)); + } + + if (tenantId) { + headers.set('X-Tenant-Id', String(tenantId)); + } + if (userId) { + headers.set('X-User-Id', String(userId)); + } + + const fetchModels = async (path: string): Promise => + fetch(new URL(path, apiEndpoint), { + headers, + signal: AbortSignal.timeout(15_000), + }); + let response = await fetchModels('/v3/config'); + + if ([400, 404, 405].includes(response.status)) { + // Upstream splits this route by account scope: enterprise accounts must hit + // their own segment, otherwise they are served the personal model catalog. + const enterpriseScope = String(enterpriseId ?? '').trim() || 'personal'; + response = await fetchModels( + `/console/enterprises/${encodeURIComponent(enterpriseScope)}/models`, + ); + } + + if (!response.ok) { + throw new Error(`Model discovery failed with status ${response.status}`); + } + + const payload = (await response.json()) as { + code?: unknown; + data?: { + agents?: Array<{ models?: unknown; name?: unknown }>; + models?: Array<{ disabled?: unknown; id?: unknown; name?: unknown }>; + }; + }; + + if (payload.code !== 0) { + throw new Error('Model discovery returned an unsuccessful response'); + } + + const cliModels = payload.data?.agents?.find( + (agent) => agent.name === 'cli', + )?.models; + const modelsById = new Map( + (payload.data?.models ?? []).flatMap((model) => { + const id = typeof model.id === 'string' ? model.id.trim() : ''; + + if (!id || model.disabled === true) { + return []; + } + + return [ + [ + id, + { + displayName: + typeof model.name === 'string' && model.name.trim() + ? model.name + : id, + id, + }, + ] as const, + ]; + }), + ); + const declaredModelIds = new Set( + (payload.data?.models ?? []) + .map((model) => (typeof model.id === 'string' ? model.id.trim() : '')) + .filter(Boolean), + ); + + if (!Array.isArray(cliModels)) { + return []; + } + + return cliModels.flatMap((modelId) => { + if (typeof modelId !== 'string') { + return []; + } + + const model = modelsById.get(modelId); + if (!model && declaredModelIds.has(modelId)) { + return []; + } + return [ + model ?? { + displayName: modelId, + id: modelId, + }, + ]; + }); +}; + +export const getModelsForCredentials = async ( + credentials: CredentialRecord[], +): Promise => { + const settled = await Promise.allSettled( + credentials.map((credential) => { + const supportedModels = getCredentialSupportedModels(credential.data); + + if (supportedModels.length) { + return Promise.resolve( + supportedModels.map((id) => ({ displayName: id, id })), + ); + } + + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + return bearerToken + ? getModelsForCredential({ + bearerToken, + credentialData: credential.data, + }) + : Promise.resolve([]); + }), + ); + const models = new Map(); + + settled.forEach((result) => { + if (result.status !== 'fulfilled') { + return; + } + + result.value.forEach((model) => { + models.set(model.id, model); + }); + }); + + return [...models.values()].sort((left, right) => + left.id.localeCompare(right.id), + ); +}; + +export const getModelsByCredential = async ( + credentials: CredentialRecord[], +): Promise< + Record +> => { + const results = await Promise.all( + credentials.map(async (credential) => { + const bearerToken = String( + credential.data.bearer_token ?? credential.data.access_token ?? '', + ).trim(); + + try { + const models = bearerToken + ? await getModelsForCredential({ + bearerToken, + credentialData: credential.data, + }) + : []; + + return [credential.filename, { error: null, models }] as const; + } catch (error) { + return [ + credential.filename, + { + error: + error instanceof Error ? error.message : 'Model discovery failed', + models: [], + }, + ] as const; + } + }), + ); + + return Object.fromEntries(results); +}; + +export const getModelsResponse = async ( + request?: NextRequest, +): Promise => { + const accessKey = request ? await resolveRequestAccessKey(request) : null; + const models = ( + await getModelsForCredentials( + await listEligibleCredentialRecords(accessKey?.credentialFilenames), + ) + ).map((model) => ({ + id: model.id, + slug: model.id, + display_name: model.displayName, + object: 'model', + created: 0, + owned_by: 'codebuddy', + })); + + return Response.json({ + object: 'list', + data: models, + models, + }); +}; diff --git a/lib/server/proxy/codebuddy-responses-request.ts b/lib/server/proxy/codebuddy-responses-request.ts new file mode 100644 index 0000000..f2444fb --- /dev/null +++ b/lib/server/proxy/codebuddy-responses-request.ts @@ -0,0 +1,382 @@ +import { stringifyContent } from '../shared/content'; +import { resolveHyResponsesReasoning } from '../shared/hy-thought-depth'; +import type { ChatRequestBody } from './codebuddy-types'; + +export const isImageContentPart = (part: unknown): boolean => { + if (!part || typeof part !== 'object') { + return false; + } + + const value = part as { image_url?: unknown; type?: unknown }; + + if (value.type === 'image_url' || value.type === 'input_image') { + return true; + } + + // Accept the shapes an OpenAI-compatible client may send even when `type` + // is absent or unexpected: any part carrying an image URL is an image. + return ( + typeof value.image_url === 'string' || + Boolean( + value.image_url && + typeof value.image_url === 'object' && + typeof (value.image_url as { url?: unknown }).url === 'string', + ) + ); +}; + +/** + * Reads the image URL out of a Responses `input_image` / `image_url` part. + * Returns undefined when the part carries no usable URL, so callers can drop it + * rather than forwarding a block the upstream would reject. + */ +export const extractImageUrl = (part: unknown): string | undefined => { + if (!part || typeof part !== 'object') { + return undefined; + } + + const { image_url: imageUrl } = part as { image_url?: unknown }; + + // `input_image` carries a bare URL string; the OpenAI Chat-style + // `image_url` part nests it under `url`. + if (typeof imageUrl === 'string') { + return imageUrl || undefined; + } + + if ( + imageUrl && + typeof imageUrl === 'object' && + typeof (imageUrl as { url?: unknown }).url === 'string' + ) { + return (imageUrl as { url: string }).url || undefined; + } + + return undefined; +}; + +export const mapChatContentToResponses = ( + content: unknown, +): Array> => { + if (!Array.isArray(content)) { + return [ + { + text: stringifyContent(content), + type: 'input_text', + }, + ]; + } + + return content.flatMap((part): Array> => { + if (typeof part === 'string') { + return [{ text: part, type: 'input_text' }]; + } + if (!part || typeof part !== 'object') { + return [{ text: JSON.stringify(part), type: 'input_text' }]; + } + const value = part as { + image_url?: string | { detail?: unknown; url?: unknown }; + text?: unknown; + type?: unknown; + }; + if (value.type === 'image_url') { + const imageUrl = + typeof value.image_url === 'string' + ? value.image_url + : value.image_url?.url; + if (typeof imageUrl === 'string' && imageUrl) { + const detail = + typeof value.image_url === 'object' && + typeof value.image_url.detail === 'string' + ? value.image_url.detail + : undefined; + return [ + { + image_url: imageUrl, + ...(detail ? { detail } : {}), + type: 'input_image', + }, + ]; + } + } + if (value.type === 'input_image' && typeof value.image_url === 'string') { + return [{ image_url: value.image_url, type: 'input_image' }]; + } + if (typeof value.text === 'string') { + return [{ text: value.text, type: 'input_text' }]; + } + return [{ text: JSON.stringify(value), type: 'input_text' }]; + }); +}; + +export const translateChatToolChoiceToResponses = ( + toolChoice: unknown, +): unknown => { + if (typeof toolChoice === 'string') return toolChoice; + if (!toolChoice || typeof toolChoice !== 'object') return undefined; + const value = toolChoice as { + function?: { name?: unknown }; + name?: unknown; + type?: unknown; + }; + if (value.type !== 'function') return toolChoice; + const name = value.function?.name ?? value.name; + return typeof name === 'string' ? { name, type: 'function' } : toolChoice; +}; + +export const translateChatResponseFormatToResponses = ( + responseFormat: unknown, +): Record | undefined => { + if (!responseFormat || typeof responseFormat !== 'object') return undefined; + const value = responseFormat as { + json_schema?: Record; + type?: unknown; + }; + if (value.type === 'json_object') { + return { format: { type: 'json_object' } }; + } + if (value.type !== 'json_schema' || !value.json_schema) return undefined; + const schema = value.json_schema; + if (typeof schema.name !== 'string' || !schema.name) return undefined; + return { + format: { + ...(schema.description ? { description: schema.description } : {}), + name: schema.name, + schema: schema.schema ?? { type: 'object', properties: {} }, + ...(typeof schema.strict === 'boolean' ? { strict: schema.strict } : {}), + type: 'json_schema', + }, + }; +}; + +export const translateChatThinkingToResponses = ( + thinking: Record | undefined, + reasoningEffort: string | undefined, +): Record | undefined => { + if (!thinking) + return reasoningEffort ? { effort: reasoningEffort } : undefined; + + if (thinking.type === 'disabled') return { effort: 'none' }; + if (thinking.type !== 'adaptive' && thinking.type !== 'enabled') { + return undefined; + } + + const budgetTokens = + typeof thinking.budget_tokens === 'number' + ? thinking.budget_tokens + : Number.NaN; + const effort = reasoningEffort + ? reasoningEffort + : Number.isFinite(budgetTokens) + ? budgetTokens <= 2_048 + ? 'low' + : budgetTokens <= 8_192 + ? 'medium' + : 'high' + : undefined; + + return { + ...(effort ? { effort } : {}), + summary: 'auto', + }; +}; + +/** + * Codex sends `reasoning.effort` in the OpenAI vocabulary, which Hy models do + * not accept, so the effort is rewritten onto the Hy vocabulary before the body + * is forwarded. + */ +export const resolveHyResponsesBody = async ( + body: Record, +): Promise> => { + const reasoning = await resolveHyResponsesReasoning( + typeof body.model === 'string' ? body.model : undefined, + body.reasoning as Record | undefined, + ); + + return { ...body, reasoning }; +}; + +export const normalizeResponsesUpstreamBody = async ( + body: Record, +): Promise> => { + const { messages, ...rest } = body; + + if (rest.input !== undefined || !Array.isArray(messages)) { + return resolveHyResponsesBody(rest); + } + + const systemInstructions = messages + .filter((message) => { + return ( + message && + typeof message === 'object' && + ((message as { role?: unknown }).role === 'system' || + (message as { role?: unknown }).role === 'developer') + ); + }) + .map((message) => { + return stringifyContent((message as { content?: unknown }).content); + }) + .filter(Boolean) + .join('\n\n'); + const input = messages.flatMap((message) => { + if (!message || typeof message !== 'object') return []; + const value = message as { content?: unknown; role?: unknown }; + if (value.role === 'system' || value.role === 'developer') return []; + const role = value.role === 'assistant' ? 'assistant' : 'user'; + return [ + { + content: mapChatContentToResponses(value.content), + role, + }, + ]; + }); + + const existingInstructions = + typeof rest.instructions === 'string' ? rest.instructions.trim() : ''; + const instructions = [existingInstructions, systemInstructions] + .filter(Boolean) + .join('\n\n'); + + return resolveHyResponsesBody({ + ...rest, + ...(instructions ? { instructions } : {}), + input, + }); +}; + +export const buildResponsesBodyFromChat = async ( + body: ChatRequestBody, +): Promise> => { + const instructions = body.messages + ?.filter( + (message) => message.role === 'system' || message.role === 'developer', + ) + .map((message) => stringifyContent(message.content)) + .filter(Boolean) + .join('\n\n'); + const input = + body.messages + ?.filter( + (message) => message.role !== 'system' && message.role !== 'developer', + ) + .map((message) => { + if (message.role === 'tool') { + // A tool may return an image, e.g. a screenshot. The upstream + // `function_call_output` carries `output` as structured content, so + // an image part is preserved there; stringifying it would hand the + // model a base64 dump instead of the image. + const toolOutput = Array.isArray(message.content) + ? message.content.filter( + (part) => part !== null && part !== undefined, + ) + : message.content; + const hasImage = Array.isArray(toolOutput) + ? toolOutput.some(isImageContentPart) + : isImageContentPart(toolOutput); + + return { + call_id: message.tool_call_id, + output: hasImage + ? mapChatContentToResponses(toolOutput) + : stringifyContent(toolOutput), + type: 'function_call_output', + }; + } + const toolCalls = Array.isArray(message.tool_calls) + ? message.tool_calls + : []; + const functionCalls = toolCalls.flatMap((toolCall) => { + if (!toolCall || typeof toolCall !== 'object') return []; + const call = toolCall as { + function?: { arguments?: unknown; name?: unknown }; + id?: unknown; + }; + if (typeof call.function?.name !== 'string') return []; + return [ + { + arguments: String(call.function.arguments ?? ''), + call_id: String(call.id ?? crypto.randomUUID()), + name: call.function.name, + type: 'function_call', + }, + ]; + }); + const content = mapChatContentToResponses(message.content); + const hasContent = content.some((part) => { + return ( + (part.type === 'input_text' && Boolean(part.text)) || + (part.type === 'input_image' && Boolean(part.image_url)) + ); + }); + const shouldOmitMessage = + message.role === 'assistant' && + functionCalls.length > 0 && + !hasContent; + + return [ + ...(shouldOmitMessage + ? [] + : [ + { + content, + role: message.role === 'assistant' ? 'assistant' : 'user', + }, + ]), + ...functionCalls, + ]; + }) + .flat() ?? []; + const tools = body.tools?.flatMap((tool) => { + if (!tool || typeof tool !== 'object') return []; + const value = tool as { + function?: Record; + type?: unknown; + }; + const definition: Record = + value.type === 'function' && value.function ? value.function : value; + if (typeof definition.name !== 'string') return []; + return [ + { + ...definition, + parameters: definition.parameters ?? { type: 'object', properties: {} }, + type: 'function', + }, + ]; + }); + const text = translateChatResponseFormatToResponses(body.response_format); + const reasoning = await resolveHyResponsesReasoning( + body.model, + translateChatThinkingToResponses(body.thinking, body.reasoning_effort), + ); + + return { + ...(instructions ? { instructions } : {}), + input, + max_output_tokens: body.max_tokens ?? body.max_completion_tokens, + model: body.model, + parallel_tool_calls: body.parallel_tool_calls, + reasoning, + stream: Boolean(body.stream), + temperature: body.temperature, + top_p: body.top_p, + ...(tools?.length ? { tools } : {}), + ...(body.tool_choice + ? { tool_choice: translateChatToolChoiceToResponses(body.tool_choice) } + : {}), + ...(text ? { text } : {}), + }; +}; + +export const getUnsupportedResponsesChatOptions = ( + body: ChatRequestBody, +): string[] => { + return [ + body.frequency_penalty !== undefined ? 'frequency_penalty' : null, + body.presence_penalty !== undefined ? 'presence_penalty' : null, + body.thinking !== undefined && + !translateChatThinkingToResponses(body.thinking, body.reasoning_effort) + ? 'thinking' + : null, + ].filter((name): name is string => Boolean(name)); +}; diff --git a/lib/server/proxy/codebuddy-responses-response.ts b/lib/server/proxy/codebuddy-responses-response.ts new file mode 100644 index 0000000..243552d --- /dev/null +++ b/lib/server/proxy/codebuddy-responses-response.ts @@ -0,0 +1,700 @@ +import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { + chatStreamErrorChunks, + createStreamCloser, + readTimeoutFrame, + toUpstreamTimeoutMessage, +} from '../shared/upstream-timeout'; +import { + extractResponsesUsage, + mapResponsesUsageToChat, + parseUsageHeader, + recordProxyUsage, +} from './codebuddy-usage'; +import { + CORS_HEADERS, + MAX_STREAM_FRAME_LENGTH, + type ProxyContext, +} from './codebuddy-types'; + +export const normalizeStopSequences = ( + stop: string | string[] | undefined, +): string[] => { + return (Array.isArray(stop) ? stop : stop ? [stop] : []).filter(Boolean); +}; + +export const findFirstStopSequence = ( + text: string, + stopSequences: string[], +): number | null => { + return stopSequences.reduce((earliest, stopSequence) => { + const index = text.indexOf(stopSequence); + if (index < 0) return earliest; + return earliest === null ? index : Math.min(earliest, index); + }, null); +}; + +export const getPendingStopPrefixLength = ( + text: string, + stopSequences: string[], +): number => { + const maximumLength = Math.min( + text.length, + Math.max( + 0, + ...stopSequences.map((stopSequence) => stopSequence.length - 1), + ), + ); + + for (let length = maximumLength; length > 0; length -= 1) { + const suffix = text.slice(-length); + if (stopSequences.some((stopSequence) => stopSequence.startsWith(suffix))) { + return length; + } + } + + return 0; +}; + +export const extractResponsesReasoningText = (output: unknown[]): string => { + return output + .flatMap((item) => { + if (!item || typeof item !== 'object') return []; + const value = item as { + content?: unknown; + summary?: unknown; + type?: unknown; + }; + if (value.type !== 'reasoning') return []; + return [value.summary, value.content].flatMap((parts) => { + if (!Array.isArray(parts)) return []; + return parts.flatMap((part) => { + if (!part || typeof part !== 'object') return []; + const text = (part as { text?: unknown }).text; + return typeof text === 'string' ? [text] : []; + }); + }); + }) + .join(''); +}; + +export const mapResponsesPayloadToChat = ( + payload: Record, + model: string, + stop: string | string[] | undefined, +): Record => { + const output = Array.isArray(payload.output) ? payload.output : []; + const toolCalls = output.flatMap((item) => { + if (!item || typeof item !== 'object') return []; + const value = item as Record; + if ( + value.type !== 'function_call' && + value.type !== 'mcp_call' && + value.type !== 'custom_tool_call' + ) { + return []; + } + const isCustomToolCall = value.type === 'custom_tool_call'; + const customArguments = JSON.stringify({ + input: String(value.input ?? value.arguments ?? ''), + }); + return [ + { + function: { + arguments: String( + isCustomToolCall ? customArguments : (value.arguments ?? ''), + ), + name: String(value.name ?? 'function'), + }, + id: String(value.call_id ?? value.id ?? crypto.randomUUID()), + type: 'function', + }, + ]; + }); + const usage = + payload.usage && typeof payload.usage === 'object' + ? (payload.usage as Record) + : undefined; + const inputTokens = Number(usage?.input_tokens ?? 0); + const outputTokens = Number(usage?.output_tokens ?? 0); + + const rawOutputText = + typeof payload.output_text === 'string' + ? payload.output_text + : output + .flatMap((item) => { + if (!item || typeof item !== 'object') return []; + const content = (item as { content?: unknown }).content; + if (!Array.isArray(content)) return []; + return content.flatMap((part) => { + if (!part || typeof part !== 'object') return []; + const value = part as { text?: unknown; type?: unknown }; + return value.type === 'output_text' && + typeof value.text === 'string' + ? [value.text] + : []; + }); + }) + .join(''); + const stopIndex = findFirstStopSequence( + rawOutputText, + normalizeStopSequences(stop), + ); + const outputText = + stopIndex === null ? rawOutputText : rawOutputText.slice(0, stopIndex); + const reasoningText = extractResponsesReasoningText(output); + const incompleteReason = + payload.incomplete_details && typeof payload.incomplete_details === 'object' + ? (payload.incomplete_details as { reason?: unknown }).reason + : undefined; + const finishReason = + payload.status === 'incomplete' + ? incompleteReason === 'content_filter' + ? 'content_filter' + : 'length' + : toolCalls.length + ? 'tool_calls' + : 'stop'; + + return { + choices: [ + { + finish_reason: finishReason, + index: 0, + message: { + content: outputText || null, + role: 'assistant', + ...(reasoningText ? { reasoning_content: reasoningText } : {}), + ...(toolCalls.length ? { tool_calls: toolCalls } : {}), + }, + }, + ], + created: Number(payload.created_at ?? Math.floor(Date.now() / 1000)), + id: String(payload.id ?? `chatcmpl-${crypto.randomUUID()}`), + model, + object: 'chat.completion', + usage: { + completion_tokens: outputTokens, + prompt_tokens: inputTokens, + total_tokens: Number(usage?.total_tokens ?? inputTokens + outputTokens), + }, + }; +}; + +export const mapResponsesStreamToChat = ( + upstreamResponse: Response, + model: string, + proxyContext: ProxyContext, + route: string, + stop: string | string[] | undefined, + includeUsage: boolean, +): Response => { + const closer = createStreamCloser(); + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + const responseId = `chatcmpl-${crypto.randomUUID()}`; + let reader: ReadableStreamDefaultReader | null = + upstreamResponse.body?.getReader() ?? null; + const fallbackUsage = parseUsageHeader(upstreamResponse); + let buffer = ''; + let emittedFinish = false; + let emittedUsage = false; + let hasToolCalls = false; + let latestUsage = fallbackUsage; + let usageRecorded = false; + const stopSequences = normalizeStopSequences(stop); + let pendingStopText = ''; + const toolIndexes = new Map(); + const toolCallIds = new Map(); + const customToolCallIds = new Set(); + const closedCustomToolCallIds = new Set(); + let nextToolIndex = 0; + let stoppedLocally = false; + + const getToolIndex = (itemId: string): number => { + const existing = toolIndexes.get(itemId); + if (existing !== undefined) return existing; + const index = nextToolIndex; + nextToolIndex += 1; + toolIndexes.set(itemId, index); + return index; + }; + + const encodeChunk = (choice: Record): Uint8Array => { + return encoder.encode( + `data: ${JSON.stringify({ + choices: [choice], + created: Math.floor(Date.now() / 1000), + id: responseId, + model, + object: 'chat.completion.chunk', + })}\n\n`, + ); + }; + + const enqueueUsage = ( + controller: ReadableStreamDefaultController, + ): void => { + if (!includeUsage || emittedUsage) return; + const usage = mapResponsesUsageToChat(latestUsage); + if (!usage) return; + + emittedUsage = true; + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [], + created: Math.floor(Date.now() / 1000), + id: responseId, + model, + object: 'chat.completion.chunk', + usage, + })}\n\n`, + ), + ); + }; + + const recordStreamUsage = async (): Promise => { + if (usageRecorded) return; + usageRecorded = true; + try { + await recordProxyUsage({ + model, + proxyContext, + route, + usage: latestUsage, + }); + } catch (error) { + console.error('[CodeBuddy2API] Failed to record Responses stream usage', { + error, + route, + }); + } + }; + + const cancelAndReleaseReader = async (reason?: unknown): Promise => { + try { + await reader?.cancel(reason); + } catch (error) { + console.error('[CodeBuddy2API] Failed to cancel Responses stream', { + error, + route, + }); + } finally { + reader?.releaseLock(); + reader = null; + } + }; + + const emitCustomToolCallClosures = ( + controller: ReadableStreamDefaultController, + ): void => { + customToolCallIds.forEach((itemId) => { + if (closedCustomToolCallIds.has(itemId)) return; + const index = getToolIndex(itemId); + const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; + controller.enqueue( + encodeChunk({ + delta: { + tool_calls: [{ function: { arguments: '"}' }, id: callId, index }], + }, + index: 0, + }), + ); + closedCustomToolCallIds.add(itemId); + }); + }; + + const stream = new ReadableStream({ + async pull(controller) { + if (!reader) { + await recordStreamUsage(); + controller.close(); + return; + } + while (true) { + let readResult: ReadableStreamReadResult; + try { + readResult = await reader.read(); + } catch (error) { + await recordStreamUsage(); + reader.releaseLock(); + reader = null; + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage !== null) { + closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); + return; + } + + controller.error(error); + return; + } + const { done, value } = readResult; + if (done) { + if (pendingStopText) { + controller.enqueue( + encodeChunk({ + delta: { content: pendingStopText }, + index: 0, + }), + ); + pendingStopText = ''; + } + emitCustomToolCallClosures(controller); + if (!emittedFinish) { + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: hasToolCalls ? 'tool_calls' : 'stop', + index: 0, + }), + ); + } + enqueueUsage(controller); + controller.enqueue(encodeDoneFrame()); + await recordStreamUsage(); + reader.releaseLock(); + reader = null; + controller.close(); + return; + } + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split(/\r?\n\r?\n/); + buffer = frames.pop() ?? ''; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader(); + await recordStreamUsage(); + controller.close(); + return; + } + let emitted = false; + for (const frame of frames) { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + controller.enqueue( + encoder.encode( + 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', + ), + ); + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader(); + await recordStreamUsage(); + controller.close(); + return; + } + const dataLine = frame + .split(/\r?\n/) + .find((line) => line.startsWith('data: ')); + if (!dataLine || dataLine === 'data: [DONE]') continue; + // The upstream here is the chat pipeline, which reports a deadline as + // a terminal error chunk and closes cleanly. Without this the failure + // would be reported to the client as an empty successful response. + const upstreamError = readTimeoutFrame(frame); + + if (upstreamError !== null) { + closer.fail(controller, chatStreamErrorChunks(upstreamError)); + await cancelAndReleaseReader(); + await recordStreamUsage(); + return; + } + try { + const event = JSON.parse(dataLine.slice(6)) as { + delta?: unknown; + item?: unknown; + item_id?: unknown; + output_index?: unknown; + error?: unknown; + response?: unknown; + type?: unknown; + }; + latestUsage = extractResponsesUsage(event) ?? latestUsage; + if ( + stoppedLocally && + event.type !== 'response.completed' && + event.type !== 'response.incomplete' + ) { + continue; + } + if (event.type === 'response.output_text.delta') { + const delta = String(event.delta ?? ''); + if (stopSequences.length) { + pendingStopText += delta; + const stopIndex = findFirstStopSequence( + pendingStopText, + stopSequences, + ); + if (stopIndex !== null) { + const content = pendingStopText.slice(0, stopIndex); + if (content) { + controller.enqueue( + encodeChunk({ delta: { content }, index: 0 }), + ); + } + pendingStopText = ''; + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: 'stop', + index: 0, + }), + ); + emittedFinish = true; + stoppedLocally = true; + emitted = true; + continue; + } + + const pendingLength = getPendingStopPrefixLength( + pendingStopText, + stopSequences, + ); + const content = pendingStopText.slice( + 0, + pendingStopText.length - pendingLength, + ); + pendingStopText = pendingLength + ? pendingStopText.slice(-pendingLength) + : ''; + if (!content) continue; + controller.enqueue( + encodeChunk({ delta: { content }, index: 0 }), + ); + emitted = true; + continue; + } + controller.enqueue( + encodeChunk({ + delta: { content: delta }, + index: 0, + }), + ); + emitted = true; + continue; + } + if ( + event.type === 'response.reasoning_summary_text.delta' || + event.type === 'response.reasoning_text.delta' + ) { + controller.enqueue( + encodeChunk({ + delta: { reasoning_content: String(event.delta ?? '') }, + index: 0, + }), + ); + emitted = true; + continue; + } + if ( + event.type === 'response.output_item.added' && + event.item && + typeof event.item === 'object' + ) { + const item = event.item as { + arguments?: unknown; + call_id?: unknown; + id?: unknown; + input?: unknown; + name?: unknown; + type?: unknown; + }; + if ( + item.type !== 'function_call' && + item.type !== 'mcp_call' && + item.type !== 'custom_tool_call' + ) { + continue; + } + const isCustomToolCall = item.type === 'custom_tool_call'; + const initialArguments = isCustomToolCall + ? `{"input":"${JSON.stringify( + String(item.input ?? item.arguments ?? ''), + ).slice(1, -1)}` + : String(item.arguments ?? ''); + const itemId = String(item.id ?? item.call_id ?? nextToolIndex); + const index = getToolIndex(itemId); + const callId = String(item.call_id ?? item.id ?? itemId); + toolCallIds.set(itemId, callId); + if (isCustomToolCall) { + customToolCallIds.add(itemId); + } + hasToolCalls = true; + controller.enqueue( + encodeChunk({ + delta: { + tool_calls: [ + { + function: { + arguments: initialArguments, + name: String(item.name ?? 'function'), + }, + id: callId, + index, + type: 'function', + }, + ], + }, + index: 0, + }), + ); + emitted = true; + continue; + } + if ( + event.type === 'response.function_call_arguments.delta' || + event.type === 'response.mcp_call_arguments.delta' || + event.type === 'response.custom_tool_call_input.delta' + ) { + const itemId = String( + event.item_id ?? event.output_index ?? nextToolIndex, + ); + const index = getToolIndex(itemId); + const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; + toolCallIds.set(itemId, callId); + hasToolCalls = true; + const argumentDelta = + event.type === 'response.custom_tool_call_input.delta' + ? JSON.stringify(String(event.delta ?? '')).slice(1, -1) + : String(event.delta ?? ''); + controller.enqueue( + encodeChunk({ + delta: { + tool_calls: [ + { + function: { arguments: argumentDelta }, + id: callId, + index, + }, + ], + }, + index: 0, + }), + ); + emitted = true; + continue; + } + if (event.type === 'response.completed') { + if (pendingStopText) { + controller.enqueue( + encodeChunk({ + delta: { content: pendingStopText }, + index: 0, + }), + ); + pendingStopText = ''; + } + emitCustomToolCallClosures(controller); + if (!emittedFinish) { + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: hasToolCalls ? 'tool_calls' : 'stop', + index: 0, + }), + ); + } + enqueueUsage(controller); + emittedFinish = true; + emitted = true; + if (stoppedLocally) { + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader('Stop sequence matched'); + await recordStreamUsage(); + controller.close(); + return; + } + continue; + } + if (event.type === 'response.incomplete') { + if (pendingStopText) { + controller.enqueue( + encodeChunk({ + delta: { content: pendingStopText }, + index: 0, + }), + ); + pendingStopText = ''; + } + emitCustomToolCallClosures(controller); + const incompleteReason = + event.response && typeof event.response === 'object' + ? ( + (event.response as { incomplete_details?: unknown }) + .incomplete_details as { reason?: unknown } | undefined + )?.reason + : undefined; + if (!emittedFinish) { + controller.enqueue( + encodeChunk({ + delta: {}, + finish_reason: + incompleteReason === 'content_filter' + ? 'content_filter' + : 'length', + index: 0, + }), + ); + } + enqueueUsage(controller); + emittedFinish = true; + emitted = true; + if (stoppedLocally) { + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader('Stop sequence matched'); + await recordStreamUsage(); + controller.close(); + return; + } + continue; + } + if ( + event.type === 'response.failed' || + event.type === 'response.error' || + event.type === 'error' + ) { + const failure = + event.error ?? + (event.response && typeof event.response === 'object' + ? (event.response as { error?: unknown }).error + : undefined); + const message = + failure && typeof failure === 'object' + ? String( + (failure as { message?: unknown }).message ?? failure, + ) + : String(failure ?? 'Upstream Responses stream failed'); + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ error: { message } })}\n\n`, + ), + ); + controller.enqueue(encodeDoneFrame()); + await cancelAndReleaseReader(); + await recordStreamUsage(); + controller.close(); + return; + } + } catch { + // Ignore malformed upstream events and continue reading. + } + } + if (stoppedLocally) continue; + if (emitted) return; + } + }, + async cancel(reason) { + await cancelAndReleaseReader(reason); + await recordStreamUsage(); + }, + }); + + return createSseResponse(stream, { + headers: CORS_HEADERS, + status: upstreamResponse.status, + }); +}; diff --git a/lib/server/proxy/codebuddy-server-tools.ts b/lib/server/proxy/codebuddy-server-tools.ts new file mode 100644 index 0000000..b150235 --- /dev/null +++ b/lib/server/proxy/codebuddy-server-tools.ts @@ -0,0 +1,239 @@ +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../search/tool'; +import { aggregateUpstreamStream } from './codebuddy-chat-stream'; +import { type ChatStreamChunk, type StreamProbeState } from './codebuddy-types'; + +export const isServerWebToolName = (name: string): boolean => { + const normalized = normalizeToolName(name); + + return ( + normalized === normalizeToolName(WEB_SEARCH_TOOL_NAME) || + normalized === normalizeToolName(WEB_FETCH_TOOL_NAME) + ); +}; + +export const mergeToolName = (previous: string, incoming: string): string => { + if (!previous || incoming.startsWith(previous)) { + return incoming; + } + + if (!incoming || previous.endsWith(incoming)) { + return previous; + } + + return previous + incoming; +}; + +export const classifyStreamFrame = ( + frame: string, + state: StreamProbeState, + serverToolNames: string[], +): 'passthrough' | 'server-tool' | null => { + const line = frame + .split('\n') + .find((segment) => segment.startsWith('data: ')); + + if (!line) { + return null; + } + + const raw = line.slice(6).trim(); + + if (!raw || raw === '[DONE]') { + return raw === '[DONE]' ? 'passthrough' : null; + } + + try { + const chunk = JSON.parse(raw) as ChatStreamChunk; + + for (const choice of chunk.choices ?? []) { + const delta = choice.delta; + const toolCalls = delta?.tool_calls ?? []; + let hasNonServerTool = false; + + for (const [position, toolCall] of toolCalls.entries()) { + const incoming = toolCall.function?.name; + + if (typeof incoming !== 'string' || !incoming) { + continue; + } + + const key = + typeof toolCall.index === 'number' + ? `index:${toolCall.index}` + : toolCall.id + ? `id:${toolCall.id}` + : `position:${position}`; + const name = mergeToolName(state.toolNames.get(key) ?? '', incoming); + const normalized = normalizeToolName(name); + + state.toolNames.set(key, name); + + if (isServerWebToolName(name) && serverToolNames.includes(normalized)) { + return 'server-tool'; + } + + if ( + normalized && + !serverToolNames.some((serverName) => + serverName.startsWith(normalized), + ) + ) { + hasNonServerTool = true; + } + } + + if ( + delta?.content || + delta?.reasoning_content || + delta?.reasoning || + hasNonServerTool || + choice.finish_reason != null + ) { + return 'passthrough'; + } + } + } catch { + return null; + } + + return null; +}; + +export const concatenateChunks = (chunks: Uint8Array[]): ArrayBuffer => { + const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); + const combined = new Uint8Array(size); + let offset = 0; + + for (const chunk of chunks) { + combined.set(chunk, offset); + offset += chunk.byteLength; + } + + return combined.buffer; +}; + +export const createResponseWithBody = ( + body: BodyInit, + response: Response, +): Response => + new Response(body, { + headers: response.headers, + status: response.status, + statusText: response.statusText, + }); + +export const createReplayStreamResponse = ({ + chunks, + reader, + response, +}: { + chunks: Uint8Array[]; + reader: ReadableStreamDefaultReader; + response: Response; +}): Response => { + let cancelled = false; + + const stream = new ReadableStream({ + start: (controller) => { + for (const chunk of chunks) { + controller.enqueue(chunk); + } + + const pump = async (): Promise => { + while (true) { + const { done, value } = await reader.read(); + + if (cancelled) { + return; + } + + if (done) { + reader.releaseLock(); + controller.close(); + return; + } + + controller.enqueue(value); + } + }; + + void pump().catch((error) => { + if (!cancelled) { + controller.error(error); + } + }); + }, + async cancel(reason): Promise { + cancelled = true; + try { + await reader.cancel(reason); + } finally { + reader.releaseLock(); + } + }, + }); + + return createResponseWithBody(stream, response); +}; + +export const detectServerToolStream = async ( + response: Response, + fallbackModel: string, + serverToolNames: string[], +): Promise => { + if (!response.body) { + return response; + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const chunks: Uint8Array[] = []; + const state: StreamProbeState = { toolNames: new Map() }; + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + + if (done) { + reader.releaseLock(); + return createResponseWithBody(concatenateChunks(chunks), response); + } + + chunks.push(value); + buffer += decoder.decode(value, { stream: true }); + const frames = buffer.split('\n\n'); + buffer = frames.pop() ?? ''; + + for (const frame of frames) { + const classification = classifyStreamFrame(frame, state, serverToolNames); + + if (classification === 'passthrough') { + return createReplayStreamResponse({ chunks, reader, response }); + } + + if (classification === 'server-tool') { + while (true) { + const remainder = await reader.read(); + + if (remainder.done) { + reader.releaseLock(); + break; + } + + chunks.push(remainder.value); + } + + return ( + await aggregateUpstreamStream( + new Response(concatenateChunks(chunks)), + fallbackModel, + ) + ).response; + } + } + } +}; diff --git a/lib/server/proxy/codebuddy-types.ts b/lib/server/proxy/codebuddy-types.ts new file mode 100644 index 0000000..d599996 --- /dev/null +++ b/lib/server/proxy/codebuddy-types.ts @@ -0,0 +1,137 @@ +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../search/tool'; + +/** + * Chat completions are called from browser-side clients as well as servers, so + * this route's streams carry the CORS header the other protocols do not need. + */ +export const CORS_HEADERS: Record = { + 'Access-Control-Allow-Origin': '*', +}; + +export interface OpenAIMessage { + role?: string; + content?: unknown; + tool_calls?: unknown[]; + tool_call_id?: string; + /** + * Prior-turn reasoning for an assistant message. + * + * Not an OpenAI field. The CodeBuddy chat upstream accepts it on assistant + * messages and uses it to carry reasoning across turns — the same slot + * CodeBuddy's own client populates when it replays a response. It is named + * `reasoning` rather than `reasoning_content` because that one is the + * upstream's *response* field; this is the request-side counterpart. + */ + reasoning?: string; +} + +export interface CacheableTextBlock { + cache_control?: { type: 'ephemeral' }; + text: string; + type: 'text'; +} + +export const MIN_AUTO_CACHE_TEXT_LENGTH = 1024; +export const MAX_STREAM_FRAME_LENGTH = 1_000_000; +export const CODEBUDDY_CLI_VERSION = '2.137.1'; +export const CODEBUDDY_USER_AGENT = `CLI/${CODEBUDDY_CLI_VERSION} CodeBuddy/${CODEBUDDY_CLI_VERSION}`; + +export interface ChatRequestBody { + model?: string; + messages?: OpenAIMessage[]; + stream?: boolean; + stream_options?: { + include_usage?: boolean; + }; + temperature?: number; + max_tokens?: number; + max_completion_tokens?: number; + response_format?: unknown; + top_p?: number; + frequency_penalty?: number; + presence_penalty?: number; + stop?: string | string[]; + tools?: unknown[]; + tool_choice?: unknown; + parallel_tool_calls?: boolean; + thinking?: Record; + reasoning_effort?: string; +} + +export interface ChatStreamDelta { + content?: string; + role?: string; + reasoning_content?: string; + reasoning?: string; + tool_calls?: Array<{ + index?: number; + id?: string; + type?: string; + function?: { + arguments?: string; + name?: string; + }; + }>; +} + +export interface ChatStreamChunk { + id?: string; + object?: string; + created?: number; + model?: string; + usage?: unknown; + choices?: Array<{ + delta?: ChatStreamDelta; + finish_reason?: string | null; + index?: number; + }>; +} + +export type ToolCallChunk = NonNullable[number]; + +export interface ToolCallMapping { + id: string; + index: number; +} + +export interface ToolCallNormalizationState { + mappings: Map; + nextIndex: number; +} + +export interface ResolvedAuth { + type: 'bearer'; + bearerToken: string; + userId: string; + credentialData: Record; +} + +export interface ProxyContext { + accessKeyId: string | null; + accessKeyName: string | null; + auth: ResolvedAuth; + credentialFilename: string | null; + preferences: { + firstMessageRoleToSystem: boolean; + firstSystemMessageRoleToUser: boolean; + upstreamProtocol: 'chat' | 'responses'; + }; +} + +export interface DiscoveredModel { + displayName: string; + id: string; +} + +export const SERVER_WEB_TOOL_NAMES = [ + normalizeToolName(WEB_SEARCH_TOOL_NAME), + normalizeToolName(WEB_FETCH_TOOL_NAME), +]; + +export interface StreamProbeState { + toolNames: Map; +} diff --git a/lib/server/proxy/codebuddy-upstream.ts b/lib/server/proxy/codebuddy-upstream.ts new file mode 100644 index 0000000..fe73292 --- /dev/null +++ b/lib/server/proxy/codebuddy-upstream.ts @@ -0,0 +1,272 @@ +import type { NextRequest } from 'next/server'; + +import { getCodeBuddyApiEndpoint, getDefaultModel } from '../domain/config'; +import { getCredentialSupportedModels } from '../domain/credentials'; +import { resolveHyChatThinking } from '../shared/hy-thought-depth'; +import { getRequestHeaderMap } from '../shared/http'; +import { getCredentialValue } from './codebuddy-context'; +import { + type CacheableTextBlock, + type ChatRequestBody, + CODEBUDDY_CLI_VERSION, + CODEBUDDY_USER_AGENT, + MIN_AUTO_CACHE_TEXT_LENGTH, + type OpenAIMessage, + type ProxyContext, + type ResolvedAuth, +} from './codebuddy-types'; + +export const hasPromptCacheControl = (content: unknown): boolean => { + return ( + Array.isArray(content) && + content.some( + (part) => !!part && typeof part === 'object' && 'cache_control' in part, + ) + ); +}; + +export const createCacheableTextBlock = (text: string): CacheableTextBlock => ({ + type: 'text', + text, + cache_control: { type: 'ephemeral' }, +}); + +export const addPromptCacheControl = ( + message: OpenAIMessage, +): OpenAIMessage => { + if ( + typeof message.content === 'string' && + message.content.trim().length >= MIN_AUTO_CACHE_TEXT_LENGTH + ) { + return { + ...message, + content: [createCacheableTextBlock(message.content)], + }; + } + + if (Array.isArray(message.content)) { + const textIndex = message.content.findIndex( + (part) => + !!part && + typeof part === 'object' && + (part as { type?: unknown }).type === 'text' && + typeof (part as { text?: unknown }).text === 'string' && + (part as { text: string }).text.trim().length >= + MIN_AUTO_CACHE_TEXT_LENGTH, + ); + + if (textIndex >= 0) { + return { + ...message, + content: message.content.map((part, index) => + index === textIndex && part && typeof part === 'object' + ? { + ...part, + cache_control: { type: 'ephemeral' }, + } + : part, + ), + }; + } + } + + return message; +}; + +export const applyPromptCacheControl = ( + messages: OpenAIMessage[], +): OpenAIMessage[] => { + const explicitCacheControl = messages.some((message) => + hasPromptCacheControl(message.content), + ); + + if (explicitCacheControl) { + return messages; + } + + const cacheableIndexes = new Set(); + const systemIndex = messages.findIndex( + (message) => message.role === 'system', + ); + + if (systemIndex >= 0) { + cacheableIndexes.add(systemIndex); + } + + let lastUserIndex = -1; + + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index]?.role === 'user') { + lastUserIndex = index; + break; + } + } + + if (lastUserIndex >= 0) { + cacheableIndexes.add(lastUserIndex); + } + + if (cacheableIndexes.size === 0) { + return messages; + } + + return messages.map((message, index) => + cacheableIndexes.has(index) ? addPromptCacheControl(message) : message, + ); +}; + +export const normalizeMessages = ( + messages: OpenAIMessage[], + firstMessageRoleToSystem: boolean, + firstSystemMessageRoleToUser: boolean, +): OpenAIMessage[] => { + const filtered = messages.filter( + (item) => item.role && item.content !== undefined, + ); + + const firstSystemIndex = firstSystemMessageRoleToUser + ? filtered.findIndex((message) => message.role === 'system') + : -1; + const normalized = filtered.map((message, index) => { + if ( + (firstMessageRoleToSystem && message.role === 'developer') || + index === firstSystemIndex + ) { + return { ...message, role: 'user' }; + } + + return message; + }); + + // Preserve role:'tool' messages so the OpenAI-compatible upstream + // receives a valid tool_calls/tool-result pair for multi-step tool loops. + return applyPromptCacheControl(normalized); +}; + +export const buildUpstreamHeaders = async ( + request: NextRequest, + auth: ResolvedAuth, +): Promise => { + const baseUrl = new URL(await getCodeBuddyApiEndpoint()); + const incoming = getRequestHeaderMap(request.headers); + const requestId = + incoming['x-request-id'] ?? crypto.randomUUID().replaceAll('-', ''); + const conversationId = incoming['x-conversation-id'] ?? crypto.randomUUID(); + const conversationRequestId = + incoming['x-conversation-request-id'] ?? + crypto.randomUUID().replaceAll('-', ''); + const conversationMessageId = + incoming['x-conversation-message-id'] ?? + crypto.randomUUID().replaceAll('-', ''); + const headers = new Headers(incoming); + headers.set('Accept', 'application/json'); + headers.set('Authorization', `Bearer ${auth.bearerToken}`); + headers.set('Content-Type', 'application/json'); + headers.set('Host', baseUrl.host); + headers.set('User-Agent', CODEBUDDY_USER_AGENT); + headers.set('X-Agent-Intent', 'craft'); + headers.set('X-Conversation-ID', conversationId); + headers.set('X-Conversation-Message-ID', conversationMessageId); + headers.set('X-Conversation-Request-ID', conversationRequestId); + headers.set('X-IDE-Name', 'CLI'); + headers.set('X-IDE-Type', 'CLI'); + headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); + headers.set('X-Client-Platform', 'web'); + headers.set('X-Product', 'SaaS'); + headers.set('X-Product-Version', CODEBUDDY_CLI_VERSION); + headers.set('X-Request-ID', requestId); + headers.set('X-Requested-With', 'XMLHttpRequest'); + headers.set('X-User-Id', auth.userId); + headers.set('x-stainless-arch', process.arch); + headers.set('x-stainless-lang', 'js'); + headers.set('x-stainless-os', process.platform); + headers.set('x-stainless-package-version', CODEBUDDY_CLI_VERSION); + headers.set('x-stainless-retry-count', '0'); + headers.set('x-stainless-runtime', 'node'); + headers.set('x-stainless-runtime-version', process.version); + + const domain = getCredentialValue(auth.credentialData, ['domain']); + const enterpriseId = getCredentialValue(auth.credentialData, [ + 'enterprise_id', + 'enterpriseId', + ]); + const tenantId = + getCredentialValue(auth.credentialData, ['tenant_id', 'tenantId']) ?? + enterpriseId; + + if (domain) { + headers.set('X-Domain', String(domain)); + } + + if (enterpriseId) { + headers.set('X-Enterprise-Id', String(enterpriseId)); + } + + if (tenantId) { + headers.set('X-Tenant-Id', String(tenantId)); + } + + const origin = String(domain ?? '') + .toLowerCase() + .endsWith('workbuddy.ai') + ? 'https://www.workbuddy.ai' + : 'https://www.codebuddy.cn'; + headers.set('Content-Type', 'application/json'); + headers.set('Origin', origin); + headers.set('Referer', `${origin}/`); + headers.set('User-Agent', CODEBUDDY_USER_AGENT); + headers.set('X-Product', 'SaaS'); + headers.set('X-Requested-With', 'XMLHttpRequest'); + headers.set('X-IDE-Name', 'CLI'); + headers.set('X-IDE-Type', 'CLI'); + headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); + + return headers; +}; + +export const headersToRecord = ( + headers: HeadersInit, +): Record => { + return Object.fromEntries(new Headers(headers).entries()); +}; + +export const buildUpstreamBody = async ( + body: ChatRequestBody, + context: ProxyContext, +): Promise => { + const normalizedMessages = normalizeMessages( + body.messages ?? [], + context.preferences.firstMessageRoleToSystem, + context.preferences.firstSystemMessageRoleToUser, + ); + const maxTokens = body.max_tokens ?? body.max_completion_tokens; + const credentialModels = getCredentialSupportedModels( + context.auth.credentialData, + ); + const model = + typeof body.model === 'string' && body.model.trim() + ? body.model + : (credentialModels[0] ?? (await getDefaultModel())); + + const hyThinking = await resolveHyChatThinking(model, body); + + return { + model, + messages: normalizedMessages, + stream: true, + temperature: body.temperature, + max_tokens: maxTokens, + max_completion_tokens: body.max_completion_tokens ?? maxTokens, + response_format: body.response_format, + top_p: body.top_p, + frequency_penalty: body.frequency_penalty, + presence_penalty: body.presence_penalty, + stop: body.stop, + stream_options: body.stream_options, + tools: body.tools, + tool_choice: body.tool_choice, + parallel_tool_calls: body.parallel_tool_calls, + thinking: hyThinking.thinking, + reasoning_effort: hyThinking.reasoningEffort, + }; +}; diff --git a/lib/server/proxy/codebuddy-usage-stream.ts b/lib/server/proxy/codebuddy-usage-stream.ts new file mode 100644 index 0000000..cf053ad --- /dev/null +++ b/lib/server/proxy/codebuddy-usage-stream.ts @@ -0,0 +1,192 @@ +import { + createStreamCloser, + responsesStreamErrorChunks, + toUpstreamTimeoutMessage, +} from '../shared/upstream-timeout'; +import { + extractResponsesId, + extractResponsesUsage, + recordProxyUsage, +} from './codebuddy-usage'; +import { MAX_STREAM_FRAME_LENGTH, type ProxyContext } from './codebuddy-types'; + +export const trackResponsesUsageStream = async ({ + fallbackUsage, + model, + onResponseId, + proxyContext, + upstreamResponse, +}: { + fallbackUsage: unknown; + model: string; + onResponseId?: (responseId: string) => Promise; + proxyContext: ProxyContext; + upstreamResponse: Response; +}): Promise => { + if (!upstreamResponse.body) { + await recordProxyUsage({ + model, + proxyContext, + route: '/v1/responses', + usage: fallbackUsage, + }); + + return new Response(null, { + headers: upstreamResponse.headers, + status: upstreamResponse.status, + }); + } + + const decoder = new TextDecoder(); + const encoder = new TextEncoder(); + let reader: ReadableStreamDefaultReader | null = null; + let cancelled = false; + const closer = createStreamCloser(); + let latestUsage = fallbackUsage; + let responseBinding: Promise | null = null; + let usageRecorded = false; + const releaseReader = (): void => { + reader?.releaseLock(); + reader = null; + }; + const recordStreamUsage = async (): Promise => { + if (usageRecorded) return; + usageRecorded = true; + try { + await recordProxyUsage({ + model, + proxyContext, + route: '/v1/responses', + usage: latestUsage, + }); + } catch (error) { + console.error('[CodeBuddy2API] Failed to record Responses stream usage', { + error, + route: '/v1/responses', + }); + } + }; + const bindResponseId = (id: string): Promise => { + if (!onResponseId) return Promise.resolve(); + responseBinding ??= onResponseId(id).catch((error) => { + console.error( + '[CodeBuddy2API] Failed to bind upstream Responses session', + { + error, + responseId: id, + }, + ); + }); + return responseBinding; + }; + const stream = new ReadableStream({ + start: (controller) => { + const upstreamReader = upstreamResponse.body!.getReader(); + reader = upstreamReader; + let buffer = ''; + let responseId: string | null = null; + + const inspectFrame = async (frame: string): Promise => { + for (const line of frame.split('\n')) { + if (!line.startsWith('data:')) { + continue; + } + + const raw = line.slice(5).trim(); + + if (!raw || raw === '[DONE]') { + continue; + } + + try { + const event = JSON.parse(raw) as unknown; + latestUsage = extractResponsesUsage(event) ?? latestUsage; + responseId = extractResponsesId(event) ?? responseId; + if (responseId) await bindResponseId(responseId); + } catch { + // Preserve malformed upstream frames without recording them. + } + } + }; + + const pump = async (): Promise => { + while (true) { + const { done, value } = await upstreamReader.read(); + + if (cancelled) { + return; + } + + if (done) { + if (buffer) { + await inspectFrame(buffer); + if (closer.closed) return; + controller.enqueue(encoder.encode(buffer)); + } + + await recordStreamUsage(); + await responseBinding; + releaseReader(); + closer.mark(); + controller.close(); + return; + } + + const text = decoder.decode(value, { stream: true }); + buffer += text; + const frames = buffer.split('\n\n'); + buffer = frames.pop()!; + if (buffer.length > MAX_STREAM_FRAME_LENGTH) { + buffer = ''; + } + + for (const frame of frames) { + if (frame.length > MAX_STREAM_FRAME_LENGTH) { + continue; + } + await inspectFrame(frame); + if (cancelled) return; + controller.enqueue(encoder.encode(`${frame}\n\n`)); + } + } + }; + + void pump().catch(async (error) => { + if (cancelled) return; + console.error('[CodeBuddy2API] Responses upstream stream failed', { + error, + route: '/v1/responses', + }); + await responseBinding; + await recordStreamUsage(); + releaseReader(); + const timeoutMessage = toUpstreamTimeoutMessage(error); + + if (timeoutMessage !== null) { + // Frames here are forwarded verbatim, so the error has to arrive as + // the Responses protocol's own event. + closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); + return; + } + + controller.error(error); + }); + }, + async cancel(reason): Promise { + cancelled = true; + closer.mark(); + try { + await reader?.cancel(reason); + } finally { + await responseBinding; + await recordStreamUsage(); + releaseReader(); + } + }, + }); + + return new Response(stream, { + headers: upstreamResponse.headers, + status: upstreamResponse.status, + }); +}; diff --git a/lib/server/proxy/codebuddy-usage.ts b/lib/server/proxy/codebuddy-usage.ts new file mode 100644 index 0000000..ece4b71 --- /dev/null +++ b/lib/server/proxy/codebuddy-usage.ts @@ -0,0 +1,152 @@ +import { recordUsageEvent, type UsageSnapshot } from '../domain/usage'; +import type { ProxyContext } from './codebuddy-types'; + +export const toUsageSnapshot = (usage: unknown): UsageSnapshot | null => { + if (!usage || typeof usage !== 'object') { + return null; + } + + return usage as UsageSnapshot; +}; + +export const recordProxyUsage = async ({ + model, + proxyContext, + route, + usage, +}: { + model: string; + proxyContext: ProxyContext; + route: string; + usage: unknown; +}): Promise => { + await recordUsageEvent({ + accessKeyId: proxyContext.accessKeyId, + accessKeyName: proxyContext.accessKeyName, + credentialFilename: proxyContext.credentialFilename, + model, + route, + usage: toUsageSnapshot(usage) ?? {}, + }); +}; + +export const extractResponsesUsage = (value: unknown): unknown => { + if (!value || typeof value !== 'object') { + return null; + } + + const payload = value as { + response?: { + usage?: unknown; + }; + usage?: unknown; + }; + + return payload.response?.usage ?? payload.usage ?? null; +}; + +export const mapResponsesUsageToChat = ( + usage: unknown, +): Record | null => { + if (!usage || typeof usage !== 'object') return null; + + const value = usage as { + cache_creation_input_tokens?: unknown; + cache_read_input_tokens?: unknown; + input_tokens?: unknown; + input_tokens_details?: { + cache_creation_tokens?: unknown; + cached_tokens?: unknown; + }; + output_tokens?: unknown; + output_tokens_details?: { + reasoning_tokens?: unknown; + }; + total_tokens?: unknown; + }; + const inputTokens = Number(value.input_tokens ?? 0); + const outputTokens = Number(value.output_tokens ?? 0); + const cachedTokens = Number( + value.input_tokens_details?.cached_tokens ?? + value.cache_read_input_tokens ?? + 0, + ); + const cacheCreationTokens = Number( + value.input_tokens_details?.cache_creation_tokens ?? + value.cache_creation_input_tokens ?? + 0, + ); + const reasoningTokens = Number( + value.output_tokens_details?.reasoning_tokens ?? 0, + ); + + return { + completion_tokens: outputTokens, + completion_tokens_details: { + reasoning_tokens: reasoningTokens, + }, + prompt_tokens: inputTokens, + prompt_tokens_details: { + cache_creation_tokens: cacheCreationTokens, + cached_tokens: cachedTokens, + }, + total_tokens: Number(value.total_tokens ?? inputTokens + outputTokens), + }; +}; + +export const extractResponsesId = (value: unknown): string | null => { + if (!value || typeof value !== 'object') return null; + const payload = value as { + id?: unknown; + response?: { id?: unknown }; + }; + const id = payload.response?.id ?? payload.id; + return typeof id === 'string' && id ? id : null; +}; + +export const parseUsageHeader = (response: Response): unknown => { + const usageHeader = response.headers.get('x-codebuddy-usage'); + + if (!usageHeader) { + return null; + } + + try { + return JSON.parse(usageHeader) as unknown; + } catch { + return null; + } +}; + +export const logUpstreamFailure = ({ + detail, + error, + route, + status, + url, +}: { + detail?: string; + error?: unknown; + route: string; + status?: number; + url: string; +}): void => { + const payload: Record = { + route, + url, + }; + + if (typeof status === 'number') { + payload.status = status; + } + + if (detail) { + payload.detail = detail.slice(0, 1000); + } + + if (error) { + payload.error = error; + } + + console.error('[CodeBuddy2API] Upstream request failed', payload); +}; diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index ee3916d..d7dfdb3 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -1,2893 +1,66 @@ import type { NextRequest } from 'next/server'; -import { resolveRequestAccessKey } from './auth'; import { - getApiFirstDeltaTimeoutMs, - getCodeBuddyApiEndpoint, - getDefaultModel, -} from '../domain/config'; -import { - type CredentialData, - type CredentialRecord, - findEligibleCredentialRecordByFilename, - findCredentialRecordByFilename, - getCredentialSupportedModels, - getCredentialProxySettings, - listEligibleCredentialRecords, - resolveCredentialForRequest, -} from '../domain/credentials'; -import { - enqueueUpstreamResponseSnapshot, - setDebugTraceCredential, - setDebugTraceError, - setDebugUpstreamRequest, - type DebugTrace, -} from '../domain/debug'; -import { createErrorResponse, getRequestHeaderMap } from '../shared/http'; -import { - resolveHyChatThinking, - resolveHyResponsesReasoning, -} from '../shared/hy-thought-depth'; -import { withCodeBuddyToken } from '../search/token'; -import { - normalizeToolName, - WEB_FETCH_TOOL_NAME, - WEB_SEARCH_TOOL_NAME, -} from '../search/tool'; -import { - attachServerToolExecutions, - attachServerToolTurns, - type ChatCompletionPayload, - executeWebSearchLoop, - type ServerToolCallbacks, - synthesizeChatCompletionStream, -} from './web-search-loop'; -import { - chatStreamErrorChunks, - createStreamCloser, - fetchWithDeadline, - responsesStreamErrorChunks, - readTimeoutFrame, - toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { recordUsageEvent, type UsageSnapshot } from '../domain/usage'; -import { stringifyContent } from '../shared/content'; -import { createSseResponse, encodeDoneFrame } from '../shared/sse'; - -/** - * Chat completions are called from browser-side clients as well as servers, so - * this route's streams carry the CORS header the other protocols do not need. - */ -const CORS_HEADERS: Record = { - 'Access-Control-Allow-Origin': '*', -}; - -interface OpenAIMessage { - role?: string; - content?: unknown; - tool_calls?: unknown[]; - tool_call_id?: string; - /** - * Prior-turn reasoning for an assistant message. - * - * Not an OpenAI field. The CodeBuddy chat upstream accepts it on assistant - * messages and uses it to carry reasoning across turns — the same slot - * CodeBuddy's own client populates when it replays a response. It is named - * `reasoning` rather than `reasoning_content` because that one is the - * upstream's *response* field; this is the request-side counterpart. - */ - reasoning?: string; -} - -interface CacheableTextBlock { - cache_control?: { type: 'ephemeral' }; - text: string; - type: 'text'; -} - -const MIN_AUTO_CACHE_TEXT_LENGTH = 1024; -const MAX_STREAM_FRAME_LENGTH = 1_000_000; -const CODEBUDDY_CLI_VERSION = '2.137.1'; -const CODEBUDDY_USER_AGENT = `CLI/${CODEBUDDY_CLI_VERSION} CodeBuddy/${CODEBUDDY_CLI_VERSION}`; - -export interface ChatRequestBody { - model?: string; - messages?: OpenAIMessage[]; - stream?: boolean; - stream_options?: { - include_usage?: boolean; - }; - temperature?: number; - max_tokens?: number; - max_completion_tokens?: number; - response_format?: unknown; - top_p?: number; - frequency_penalty?: number; - presence_penalty?: number; - stop?: string | string[]; - tools?: unknown[]; - tool_choice?: unknown; - parallel_tool_calls?: boolean; - thinking?: Record; - reasoning_effort?: string; -} - -interface ChatStreamDelta { - content?: string; - role?: string; - reasoning_content?: string; - reasoning?: string; - tool_calls?: Array<{ - index?: number; - id?: string; - type?: string; - function?: { - arguments?: string; - name?: string; - }; - }>; -} - -interface ChatStreamChunk { - id?: string; - object?: string; - created?: number; - model?: string; - usage?: unknown; - choices?: Array<{ - delta?: ChatStreamDelta; - finish_reason?: string | null; - index?: number; - }>; -} - -type ToolCallChunk = NonNullable[number]; - -interface ToolCallMapping { - id: string; - index: number; -} - -interface ToolCallNormalizationState { - mappings: Map; - nextIndex: number; -} - -interface ResolvedAuth { - type: 'bearer'; - bearerToken: string; - userId: string; - credentialData: Record; -} - -export interface ProxyContext { - accessKeyId: string | null; - accessKeyName: string | null; - auth: ResolvedAuth; - credentialFilename: string | null; - preferences: { - firstMessageRoleToSystem: boolean; - firstSystemMessageRoleToUser: boolean; - upstreamProtocol: 'chat' | 'responses'; - }; -} - -export interface DiscoveredModel { - displayName: string; - id: string; -} - -const getCredentialAffinityKey = ( - request: NextRequest, - accessKeyId: string | null, -): string | undefined => { - const incoming = getRequestHeaderMap(request.headers); - const conversationId = incoming['x-conversation-id']?.trim(); - - if (!conversationId) { - return undefined; - } - - if (accessKeyId) { - return `access-key:${accessKeyId}:conversation:${conversationId}`; - } - - return `global:conversation:${conversationId}`; -}; - -const toUsageSnapshot = (usage: unknown): UsageSnapshot | null => { - if (!usage || typeof usage !== 'object') { - return null; - } - - return usage as UsageSnapshot; -}; - -const recordProxyUsage = async ({ - model, - proxyContext, - route, - usage, -}: { - model: string; - proxyContext: ProxyContext; - route: string; - usage: unknown; -}): Promise => { - await recordUsageEvent({ - accessKeyId: proxyContext.accessKeyId, - accessKeyName: proxyContext.accessKeyName, - credentialFilename: proxyContext.credentialFilename, - model, - route, - usage: toUsageSnapshot(usage) ?? {}, - }); -}; - -const extractResponsesUsage = (value: unknown): unknown => { - if (!value || typeof value !== 'object') { - return null; - } - - const payload = value as { - response?: { - usage?: unknown; - }; - usage?: unknown; - }; - - return payload.response?.usage ?? payload.usage ?? null; -}; - -const mapResponsesUsageToChat = ( - usage: unknown, -): Record | null => { - if (!usage || typeof usage !== 'object') return null; - - const value = usage as { - cache_creation_input_tokens?: unknown; - cache_read_input_tokens?: unknown; - input_tokens?: unknown; - input_tokens_details?: { - cache_creation_tokens?: unknown; - cached_tokens?: unknown; - }; - output_tokens?: unknown; - output_tokens_details?: { - reasoning_tokens?: unknown; - }; - total_tokens?: unknown; - }; - const inputTokens = Number(value.input_tokens ?? 0); - const outputTokens = Number(value.output_tokens ?? 0); - const cachedTokens = Number( - value.input_tokens_details?.cached_tokens ?? - value.cache_read_input_tokens ?? - 0, - ); - const cacheCreationTokens = Number( - value.input_tokens_details?.cache_creation_tokens ?? - value.cache_creation_input_tokens ?? - 0, - ); - const reasoningTokens = Number( - value.output_tokens_details?.reasoning_tokens ?? 0, - ); - - return { - completion_tokens: outputTokens, - completion_tokens_details: { - reasoning_tokens: reasoningTokens, - }, - prompt_tokens: inputTokens, - prompt_tokens_details: { - cache_creation_tokens: cacheCreationTokens, - cached_tokens: cachedTokens, - }, - total_tokens: Number(value.total_tokens ?? inputTokens + outputTokens), - }; -}; - -const extractResponsesId = (value: unknown): string | null => { - if (!value || typeof value !== 'object') return null; - const payload = value as { - id?: unknown; - response?: { id?: unknown }; - }; - const id = payload.response?.id ?? payload.id; - return typeof id === 'string' && id ? id : null; -}; - -const parseUsageHeader = (response: Response): unknown => { - const usageHeader = response.headers.get('x-codebuddy-usage'); - - if (!usageHeader) { - return null; - } - - try { - return JSON.parse(usageHeader) as unknown; - } catch { - return null; - } -}; - -const trackResponsesUsageStream = async ({ - fallbackUsage, - model, - onResponseId, - proxyContext, - upstreamResponse, -}: { - fallbackUsage: unknown; - model: string; - onResponseId?: (responseId: string) => Promise; - proxyContext: ProxyContext; - upstreamResponse: Response; -}): Promise => { - if (!upstreamResponse.body) { - await recordProxyUsage({ - model, - proxyContext, - route: '/v1/responses', - usage: fallbackUsage, - }); - - return new Response(null, { - headers: upstreamResponse.headers, - status: upstreamResponse.status, - }); - } - - const decoder = new TextDecoder(); - const encoder = new TextEncoder(); - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - const closer = createStreamCloser(); - let latestUsage = fallbackUsage; - let responseBinding: Promise | null = null; - let usageRecorded = false; - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - const recordStreamUsage = async (): Promise => { - if (usageRecorded) return; - usageRecorded = true; - try { - await recordProxyUsage({ - model, - proxyContext, - route: '/v1/responses', - usage: latestUsage, - }); - } catch (error) { - console.error('[CodeBuddy2API] Failed to record Responses stream usage', { - error, - route: '/v1/responses', - }); - } - }; - const bindResponseId = (id: string): Promise => { - if (!onResponseId) return Promise.resolve(); - responseBinding ??= onResponseId(id).catch((error) => { - console.error( - '[CodeBuddy2API] Failed to bind upstream Responses session', - { - error, - responseId: id, - }, - ); - }); - return responseBinding; - }; - const stream = new ReadableStream({ - start: (controller) => { - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - let responseId: string | null = null; - - const inspectFrame = async (frame: string): Promise => { - for (const line of frame.split('\n')) { - if (!line.startsWith('data:')) { - continue; - } - - const raw = line.slice(5).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - try { - const event = JSON.parse(raw) as unknown; - latestUsage = extractResponsesUsage(event) ?? latestUsage; - responseId = extractResponsesId(event) ?? responseId; - if (responseId) await bindResponseId(responseId); - } catch { - // Preserve malformed upstream frames without recording them. - } - } - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - if (buffer) { - await inspectFrame(buffer); - if (closer.closed) return; - controller.enqueue(encoder.encode(buffer)); - } - - await recordStreamUsage(); - await responseBinding; - releaseReader(); - closer.mark(); - controller.close(); - return; - } - - const text = decoder.decode(value, { stream: true }); - buffer += text; - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - buffer = ''; - } - - for (const frame of frames) { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - continue; - } - await inspectFrame(frame); - if (cancelled) return; - controller.enqueue(encoder.encode(`${frame}\n\n`)); - } - } - }; - - void pump().catch(async (error) => { - if (cancelled) return; - console.error('[CodeBuddy2API] Responses upstream stream failed', { - error, - route: '/v1/responses', - }); - await responseBinding; - await recordStreamUsage(); - releaseReader(); - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage !== null) { - // Frames here are forwarded verbatim, so the error has to arrive as - // the Responses protocol's own event. - closer.fail(controller, responsesStreamErrorChunks(timeoutMessage)); - return; - } - - controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - await responseBinding; - await recordStreamUsage(); - releaseReader(); - } - }, - }); - - return new Response(stream, { - headers: upstreamResponse.headers, - status: upstreamResponse.status, - }); -}; - -const logUpstreamFailure = ({ - detail, - error, - route, - status, - url, -}: { - detail?: string; - error?: unknown; - route: string; - status?: number; - url: string; -}): void => { - const payload: Record = { - route, - url, - }; - - if (typeof status === 'number') { - payload.status = status; - } - - if (detail) { - payload.detail = detail.slice(0, 1000); - } - - if (error) { - payload.error = error; - } - - console.error('[CodeBuddy2API] Upstream request failed', payload); -}; - -const hasPromptCacheControl = (content: unknown): boolean => { - return ( - Array.isArray(content) && - content.some( - (part) => !!part && typeof part === 'object' && 'cache_control' in part, - ) - ); -}; - -const createCacheableTextBlock = (text: string): CacheableTextBlock => ({ - type: 'text', - text, - cache_control: { type: 'ephemeral' }, -}); - -const addPromptCacheControl = (message: OpenAIMessage): OpenAIMessage => { - if ( - typeof message.content === 'string' && - message.content.trim().length >= MIN_AUTO_CACHE_TEXT_LENGTH - ) { - return { - ...message, - content: [createCacheableTextBlock(message.content)], - }; - } - - if (Array.isArray(message.content)) { - const textIndex = message.content.findIndex( - (part) => - !!part && - typeof part === 'object' && - (part as { type?: unknown }).type === 'text' && - typeof (part as { text?: unknown }).text === 'string' && - (part as { text: string }).text.trim().length >= - MIN_AUTO_CACHE_TEXT_LENGTH, - ); - - if (textIndex >= 0) { - return { - ...message, - content: message.content.map((part, index) => - index === textIndex && part && typeof part === 'object' - ? { - ...part, - cache_control: { type: 'ephemeral' }, - } - : part, - ), - }; - } - } - - return message; -}; - -const applyPromptCacheControl = ( - messages: OpenAIMessage[], -): OpenAIMessage[] => { - const explicitCacheControl = messages.some((message) => - hasPromptCacheControl(message.content), - ); - - if (explicitCacheControl) { - return messages; - } - - const cacheableIndexes = new Set(); - const systemIndex = messages.findIndex( - (message) => message.role === 'system', - ); - - if (systemIndex >= 0) { - cacheableIndexes.add(systemIndex); - } - - let lastUserIndex = -1; - - for (let index = messages.length - 1; index >= 0; index -= 1) { - if (messages[index]?.role === 'user') { - lastUserIndex = index; - break; - } - } - - if (lastUserIndex >= 0) { - cacheableIndexes.add(lastUserIndex); - } - - if (cacheableIndexes.size === 0) { - return messages; - } - - return messages.map((message, index) => - cacheableIndexes.has(index) ? addPromptCacheControl(message) : message, - ); -}; - -const normalizeMessages = ( - messages: OpenAIMessage[], - firstMessageRoleToSystem: boolean, - firstSystemMessageRoleToUser: boolean, -): OpenAIMessage[] => { - const filtered = messages.filter( - (item) => item.role && item.content !== undefined, - ); - - const firstSystemIndex = firstSystemMessageRoleToUser - ? filtered.findIndex((message) => message.role === 'system') - : -1; - const normalized = filtered.map((message, index) => { - if ( - (firstMessageRoleToSystem && message.role === 'developer') || - index === firstSystemIndex - ) { - return { ...message, role: 'user' }; - } - - return message; - }); - - // Preserve role:'tool' messages so the OpenAI-compatible upstream - // receives a valid tool_calls/tool-result pair for multi-step tool loops. - return applyPromptCacheControl(normalized); -}; - -export const resolveProxyContext = async ( - request: NextRequest, - model?: string, -): Promise => { - const accessKey = await resolveRequestAccessKey(request); - const credential = await resolveCredentialForRequest({ - accessKeyId: accessKey?.id, - affinityKey: getCredentialAffinityKey(request, accessKey?.id ?? null), - allowedCredentialFilenames: accessKey?.credentialFilenames, - model, - }); - - if (!credential) { - throw new Error('No valid CodeBuddy credentials found'); - } - - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - if (!bearerToken) { - throw new Error('Saved credential does not include a bearer token'); - } - - return { - accessKeyId: accessKey?.id ?? null, - accessKeyName: accessKey?.name ?? null, - auth: { - type: 'bearer', - bearerToken, - userId: String(credential.data.user_id ?? 'unknown'), - credentialData: credential.data, - }, - credentialFilename: credential.filename, - preferences: getCredentialProxySettings(credential.data), - }; -}; - -export const createProxyContextFromCredential = ( - credential: CredentialRecord, -): ProxyContext => { - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - if (!bearerToken) { - throw new Error('Saved credential does not include a bearer token'); - } - - return { - accessKeyId: null, - accessKeyName: null, - auth: { - type: 'bearer', - bearerToken, - userId: String(credential.data.user_id ?? 'unknown'), - credentialData: credential.data, - }, - credentialFilename: credential.filename, - preferences: getCredentialProxySettings(credential.data), - }; -}; - -export const resolveProxyContextByCredentialFilename = async ( - filename: string, - options?: { - accessKey?: { - id?: string | null; - name?: string | null; - }; - allowedCredentialFilenames?: string[]; - requireEligible?: boolean; - }, -): Promise => { - const credential = options?.requireEligible - ? await findEligibleCredentialRecordByFilename( - filename, - options.allowedCredentialFilenames, - ) - : await findCredentialRecordByFilename(filename); - - if (!credential) { - throw new Error('Selected credential was not found'); - } - - return { - ...createProxyContextFromCredential(credential), - accessKeyId: options?.accessKey?.id ?? null, - accessKeyName: options?.accessKey?.name ?? null, - }; -}; - -const getCredentialValue = ( - value: unknown, - candidateKeys: string[], -): string | number | null => { - if (Array.isArray(value)) { - for (const item of value) { - const nested = getCredentialValue(item, candidateKeys); - - if (nested !== null && nested !== '') { - return nested; - } - } - - return null; - } - - if (value && typeof value === 'object') { - for (const key of candidateKeys) { - const direct = (value as Record)[key]; - - if (direct !== undefined && direct !== null && direct !== '') { - return direct as string | number; - } - } - - for (const nestedValue of Object.values(value as Record)) { - const nested = getCredentialValue(nestedValue, candidateKeys); - - if (nested !== null && nested !== '') { - return nested; - } - } - } - - return null; -}; - -export const buildUpstreamHeaders = async ( - request: NextRequest, - auth: ResolvedAuth, -): Promise => { - const baseUrl = new URL(await getCodeBuddyApiEndpoint()); - const incoming = getRequestHeaderMap(request.headers); - const requestId = - incoming['x-request-id'] ?? crypto.randomUUID().replaceAll('-', ''); - const conversationId = incoming['x-conversation-id'] ?? crypto.randomUUID(); - const conversationRequestId = - incoming['x-conversation-request-id'] ?? - crypto.randomUUID().replaceAll('-', ''); - const conversationMessageId = - incoming['x-conversation-message-id'] ?? - crypto.randomUUID().replaceAll('-', ''); - const headers = new Headers(incoming); - headers.set('Accept', 'application/json'); - headers.set('Authorization', `Bearer ${auth.bearerToken}`); - headers.set('Content-Type', 'application/json'); - headers.set('Host', baseUrl.host); - headers.set('User-Agent', CODEBUDDY_USER_AGENT); - headers.set('X-Agent-Intent', 'craft'); - headers.set('X-Conversation-ID', conversationId); - headers.set('X-Conversation-Message-ID', conversationMessageId); - headers.set('X-Conversation-Request-ID', conversationRequestId); - headers.set('X-IDE-Name', 'CLI'); - headers.set('X-IDE-Type', 'CLI'); - headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); - headers.set('X-Client-Platform', 'web'); - headers.set('X-Product', 'SaaS'); - headers.set('X-Product-Version', CODEBUDDY_CLI_VERSION); - headers.set('X-Request-ID', requestId); - headers.set('X-Requested-With', 'XMLHttpRequest'); - headers.set('X-User-Id', auth.userId); - headers.set('x-stainless-arch', process.arch); - headers.set('x-stainless-lang', 'js'); - headers.set('x-stainless-os', process.platform); - headers.set('x-stainless-package-version', CODEBUDDY_CLI_VERSION); - headers.set('x-stainless-retry-count', '0'); - headers.set('x-stainless-runtime', 'node'); - headers.set('x-stainless-runtime-version', process.version); - - const domain = getCredentialValue(auth.credentialData, ['domain']); - const enterpriseId = getCredentialValue(auth.credentialData, [ - 'enterprise_id', - 'enterpriseId', - ]); - const tenantId = - getCredentialValue(auth.credentialData, ['tenant_id', 'tenantId']) ?? - enterpriseId; - - if (domain) { - headers.set('X-Domain', String(domain)); - } - - if (enterpriseId) { - headers.set('X-Enterprise-Id', String(enterpriseId)); - } - - if (tenantId) { - headers.set('X-Tenant-Id', String(tenantId)); - } - - const origin = String(domain ?? '') - .toLowerCase() - .endsWith('workbuddy.ai') - ? 'https://www.workbuddy.ai' - : 'https://www.codebuddy.cn'; - headers.set('Content-Type', 'application/json'); - headers.set('Origin', origin); - headers.set('Referer', `${origin}/`); - headers.set('User-Agent', CODEBUDDY_USER_AGENT); - headers.set('X-Product', 'SaaS'); - headers.set('X-Requested-With', 'XMLHttpRequest'); - headers.set('X-IDE-Name', 'CLI'); - headers.set('X-IDE-Type', 'CLI'); - headers.set('X-IDE-Version', CODEBUDDY_CLI_VERSION); - - return headers; -}; - -const headersToRecord = (headers: HeadersInit): Record => { - return Object.fromEntries(new Headers(headers).entries()); -}; - -const buildUpstreamBody = async ( - body: ChatRequestBody, - context: ProxyContext, -): Promise => { - const normalizedMessages = normalizeMessages( - body.messages ?? [], - context.preferences.firstMessageRoleToSystem, - context.preferences.firstSystemMessageRoleToUser, - ); - const maxTokens = body.max_tokens ?? body.max_completion_tokens; - const credentialModels = getCredentialSupportedModels( - context.auth.credentialData, - ); - const model = - typeof body.model === 'string' && body.model.trim() - ? body.model - : (credentialModels[0] ?? (await getDefaultModel())); - - const hyThinking = await resolveHyChatThinking(model, body); - - return { - model, - messages: normalizedMessages, - stream: true, - temperature: body.temperature, - max_tokens: maxTokens, - max_completion_tokens: body.max_completion_tokens ?? maxTokens, - response_format: body.response_format, - top_p: body.top_p, - frequency_penalty: body.frequency_penalty, - presence_penalty: body.presence_penalty, - stop: body.stop, - stream_options: body.stream_options, - tools: body.tools, - tool_choice: body.tool_choice, - parallel_tool_calls: body.parallel_tool_calls, - thinking: hyThinking.thinking, - reasoning_effort: hyThinking.reasoningEffort, - }; -}; - -export const isImageContentPart = (part: unknown): boolean => { - if (!part || typeof part !== 'object') { - return false; - } - - const value = part as { image_url?: unknown; type?: unknown }; - - if (value.type === 'image_url' || value.type === 'input_image') { - return true; - } - - // Accept the shapes an OpenAI-compatible client may send even when `type` - // is absent or unexpected: any part carrying an image URL is an image. - return ( - typeof value.image_url === 'string' || - Boolean( - value.image_url && - typeof value.image_url === 'object' && - typeof (value.image_url as { url?: unknown }).url === 'string', - ) - ); -}; - -/** - * Reads the image URL out of a Responses `input_image` / `image_url` part. - * Returns undefined when the part carries no usable URL, so callers can drop it - * rather than forwarding a block the upstream would reject. - */ -export const extractImageUrl = (part: unknown): string | undefined => { - if (!part || typeof part !== 'object') { - return undefined; - } - - const { image_url: imageUrl } = part as { image_url?: unknown }; - - // `input_image` carries a bare URL string; the OpenAI Chat-style - // `image_url` part nests it under `url`. - if (typeof imageUrl === 'string') { - return imageUrl || undefined; - } - - if ( - imageUrl && - typeof imageUrl === 'object' && - typeof (imageUrl as { url?: unknown }).url === 'string' - ) { - return (imageUrl as { url: string }).url || undefined; - } - - return undefined; -}; - -const mapChatContentToResponses = ( - content: unknown, -): Array> => { - if (!Array.isArray(content)) { - return [ - { - text: stringifyContent(content), - type: 'input_text', - }, - ]; - } - - return content.flatMap((part): Array> => { - if (typeof part === 'string') { - return [{ text: part, type: 'input_text' }]; - } - if (!part || typeof part !== 'object') { - return [{ text: JSON.stringify(part), type: 'input_text' }]; - } - const value = part as { - image_url?: string | { detail?: unknown; url?: unknown }; - text?: unknown; - type?: unknown; - }; - if (value.type === 'image_url') { - const imageUrl = - typeof value.image_url === 'string' - ? value.image_url - : value.image_url?.url; - if (typeof imageUrl === 'string' && imageUrl) { - const detail = - typeof value.image_url === 'object' && - typeof value.image_url.detail === 'string' - ? value.image_url.detail - : undefined; - return [ - { - image_url: imageUrl, - ...(detail ? { detail } : {}), - type: 'input_image', - }, - ]; - } - } - if (value.type === 'input_image' && typeof value.image_url === 'string') { - return [{ image_url: value.image_url, type: 'input_image' }]; - } - if (typeof value.text === 'string') { - return [{ text: value.text, type: 'input_text' }]; - } - return [{ text: JSON.stringify(value), type: 'input_text' }]; - }); -}; - -const translateChatToolChoiceToResponses = (toolChoice: unknown): unknown => { - if (typeof toolChoice === 'string') return toolChoice; - if (!toolChoice || typeof toolChoice !== 'object') return undefined; - const value = toolChoice as { - function?: { name?: unknown }; - name?: unknown; - type?: unknown; - }; - if (value.type !== 'function') return toolChoice; - const name = value.function?.name ?? value.name; - return typeof name === 'string' ? { name, type: 'function' } : toolChoice; -}; - -const translateChatResponseFormatToResponses = ( - responseFormat: unknown, -): Record | undefined => { - if (!responseFormat || typeof responseFormat !== 'object') return undefined; - const value = responseFormat as { - json_schema?: Record; - type?: unknown; - }; - if (value.type === 'json_object') { - return { format: { type: 'json_object' } }; - } - if (value.type !== 'json_schema' || !value.json_schema) return undefined; - const schema = value.json_schema; - if (typeof schema.name !== 'string' || !schema.name) return undefined; - return { - format: { - ...(schema.description ? { description: schema.description } : {}), - name: schema.name, - schema: schema.schema ?? { type: 'object', properties: {} }, - ...(typeof schema.strict === 'boolean' ? { strict: schema.strict } : {}), - type: 'json_schema', - }, - }; -}; - -const translateChatThinkingToResponses = ( - thinking: Record | undefined, - reasoningEffort: string | undefined, -): Record | undefined => { - if (!thinking) - return reasoningEffort ? { effort: reasoningEffort } : undefined; - - if (thinking.type === 'disabled') return { effort: 'none' }; - if (thinking.type !== 'adaptive' && thinking.type !== 'enabled') { - return undefined; - } - - const budgetTokens = - typeof thinking.budget_tokens === 'number' - ? thinking.budget_tokens - : Number.NaN; - const effort = reasoningEffort - ? reasoningEffort - : Number.isFinite(budgetTokens) - ? budgetTokens <= 2_048 - ? 'low' - : budgetTokens <= 8_192 - ? 'medium' - : 'high' - : undefined; - - return { - ...(effort ? { effort } : {}), - summary: 'auto', - }; -}; - -const normalizeStopSequences = ( - stop: string | string[] | undefined, -): string[] => { - return (Array.isArray(stop) ? stop : stop ? [stop] : []).filter(Boolean); -}; - -const findFirstStopSequence = ( - text: string, - stopSequences: string[], -): number | null => { - return stopSequences.reduce((earliest, stopSequence) => { - const index = text.indexOf(stopSequence); - if (index < 0) return earliest; - return earliest === null ? index : Math.min(earliest, index); - }, null); -}; - -const getPendingStopPrefixLength = ( - text: string, - stopSequences: string[], -): number => { - const maximumLength = Math.min( - text.length, - Math.max( - 0, - ...stopSequences.map((stopSequence) => stopSequence.length - 1), - ), - ); - - for (let length = maximumLength; length > 0; length -= 1) { - const suffix = text.slice(-length); - if (stopSequences.some((stopSequence) => stopSequence.startsWith(suffix))) { - return length; - } - } - - return 0; -}; - -/** - * Codex sends `reasoning.effort` in the OpenAI vocabulary, which Hy models do - * not accept, so the effort is rewritten onto the Hy vocabulary before the body - * is forwarded. - */ -const resolveHyResponsesBody = async ( - body: Record, -): Promise> => { - const reasoning = await resolveHyResponsesReasoning( - typeof body.model === 'string' ? body.model : undefined, - body.reasoning as Record | undefined, - ); - - return { ...body, reasoning }; -}; - -const normalizeResponsesUpstreamBody = async ( - body: Record, -): Promise> => { - const { messages, ...rest } = body; - - if (rest.input !== undefined || !Array.isArray(messages)) { - return resolveHyResponsesBody(rest); - } - - const systemInstructions = messages - .filter((message) => { - return ( - message && - typeof message === 'object' && - ((message as { role?: unknown }).role === 'system' || - (message as { role?: unknown }).role === 'developer') - ); - }) - .map((message) => { - return stringifyContent((message as { content?: unknown }).content); - }) - .filter(Boolean) - .join('\n\n'); - const input = messages.flatMap((message) => { - if (!message || typeof message !== 'object') return []; - const value = message as { content?: unknown; role?: unknown }; - if (value.role === 'system' || value.role === 'developer') return []; - const role = value.role === 'assistant' ? 'assistant' : 'user'; - return [ - { - content: mapChatContentToResponses(value.content), - role, - }, - ]; - }); - - const existingInstructions = - typeof rest.instructions === 'string' ? rest.instructions.trim() : ''; - const instructions = [existingInstructions, systemInstructions] - .filter(Boolean) - .join('\n\n'); - - return resolveHyResponsesBody({ - ...rest, - ...(instructions ? { instructions } : {}), - input, - }); -}; - -const buildResponsesBodyFromChat = async ( - body: ChatRequestBody, -): Promise> => { - const instructions = body.messages - ?.filter( - (message) => message.role === 'system' || message.role === 'developer', - ) - .map((message) => stringifyContent(message.content)) - .filter(Boolean) - .join('\n\n'); - const input = - body.messages - ?.filter( - (message) => message.role !== 'system' && message.role !== 'developer', - ) - .map((message) => { - if (message.role === 'tool') { - // A tool may return an image, e.g. a screenshot. The upstream - // `function_call_output` carries `output` as structured content, so - // an image part is preserved there; stringifying it would hand the - // model a base64 dump instead of the image. - const toolOutput = Array.isArray(message.content) - ? message.content.filter( - (part) => part !== null && part !== undefined, - ) - : message.content; - const hasImage = Array.isArray(toolOutput) - ? toolOutput.some(isImageContentPart) - : isImageContentPart(toolOutput); - - return { - call_id: message.tool_call_id, - output: hasImage - ? mapChatContentToResponses(toolOutput) - : stringifyContent(toolOutput), - type: 'function_call_output', - }; - } - const toolCalls = Array.isArray(message.tool_calls) - ? message.tool_calls - : []; - const functionCalls = toolCalls.flatMap((toolCall) => { - if (!toolCall || typeof toolCall !== 'object') return []; - const call = toolCall as { - function?: { arguments?: unknown; name?: unknown }; - id?: unknown; - }; - if (typeof call.function?.name !== 'string') return []; - return [ - { - arguments: String(call.function.arguments ?? ''), - call_id: String(call.id ?? crypto.randomUUID()), - name: call.function.name, - type: 'function_call', - }, - ]; - }); - const content = mapChatContentToResponses(message.content); - const hasContent = content.some((part) => { - return ( - (part.type === 'input_text' && Boolean(part.text)) || - (part.type === 'input_image' && Boolean(part.image_url)) - ); - }); - const shouldOmitMessage = - message.role === 'assistant' && - functionCalls.length > 0 && - !hasContent; - - return [ - ...(shouldOmitMessage - ? [] - : [ - { - content, - role: message.role === 'assistant' ? 'assistant' : 'user', - }, - ]), - ...functionCalls, - ]; - }) - .flat() ?? []; - const tools = body.tools?.flatMap((tool) => { - if (!tool || typeof tool !== 'object') return []; - const value = tool as { - function?: Record; - type?: unknown; - }; - const definition: Record = - value.type === 'function' && value.function ? value.function : value; - if (typeof definition.name !== 'string') return []; - return [ - { - ...definition, - parameters: definition.parameters ?? { type: 'object', properties: {} }, - type: 'function', - }, - ]; - }); - const text = translateChatResponseFormatToResponses(body.response_format); - const reasoning = await resolveHyResponsesReasoning( - body.model, - translateChatThinkingToResponses(body.thinking, body.reasoning_effort), - ); - - return { - ...(instructions ? { instructions } : {}), - input, - max_output_tokens: body.max_tokens ?? body.max_completion_tokens, - model: body.model, - parallel_tool_calls: body.parallel_tool_calls, - reasoning, - stream: Boolean(body.stream), - temperature: body.temperature, - top_p: body.top_p, - ...(tools?.length ? { tools } : {}), - ...(body.tool_choice - ? { tool_choice: translateChatToolChoiceToResponses(body.tool_choice) } - : {}), - ...(text ? { text } : {}), - }; -}; - -const getUnsupportedResponsesChatOptions = ( - body: ChatRequestBody, -): string[] => { - return [ - body.frequency_penalty !== undefined ? 'frequency_penalty' : null, - body.presence_penalty !== undefined ? 'presence_penalty' : null, - body.thinking !== undefined && - !translateChatThinkingToResponses(body.thinking, body.reasoning_effort) - ? 'thinking' - : null, - ].filter((name): name is string => Boolean(name)); -}; - -const extractResponsesReasoningText = (output: unknown[]): string => { - return output - .flatMap((item) => { - if (!item || typeof item !== 'object') return []; - const value = item as { - content?: unknown; - summary?: unknown; - type?: unknown; - }; - if (value.type !== 'reasoning') return []; - return [value.summary, value.content].flatMap((parts) => { - if (!Array.isArray(parts)) return []; - return parts.flatMap((part) => { - if (!part || typeof part !== 'object') return []; - const text = (part as { text?: unknown }).text; - return typeof text === 'string' ? [text] : []; - }); - }); - }) - .join(''); -}; - -const mapResponsesPayloadToChat = ( - payload: Record, - model: string, - stop: string | string[] | undefined, -): Record => { - const output = Array.isArray(payload.output) ? payload.output : []; - const toolCalls = output.flatMap((item) => { - if (!item || typeof item !== 'object') return []; - const value = item as Record; - if ( - value.type !== 'function_call' && - value.type !== 'mcp_call' && - value.type !== 'custom_tool_call' - ) { - return []; - } - const isCustomToolCall = value.type === 'custom_tool_call'; - const customArguments = JSON.stringify({ - input: String(value.input ?? value.arguments ?? ''), - }); - return [ - { - function: { - arguments: String( - isCustomToolCall ? customArguments : (value.arguments ?? ''), - ), - name: String(value.name ?? 'function'), - }, - id: String(value.call_id ?? value.id ?? crypto.randomUUID()), - type: 'function', - }, - ]; - }); - const usage = - payload.usage && typeof payload.usage === 'object' - ? (payload.usage as Record) - : undefined; - const inputTokens = Number(usage?.input_tokens ?? 0); - const outputTokens = Number(usage?.output_tokens ?? 0); - - const rawOutputText = - typeof payload.output_text === 'string' - ? payload.output_text - : output - .flatMap((item) => { - if (!item || typeof item !== 'object') return []; - const content = (item as { content?: unknown }).content; - if (!Array.isArray(content)) return []; - return content.flatMap((part) => { - if (!part || typeof part !== 'object') return []; - const value = part as { text?: unknown; type?: unknown }; - return value.type === 'output_text' && - typeof value.text === 'string' - ? [value.text] - : []; - }); - }) - .join(''); - const stopIndex = findFirstStopSequence( - rawOutputText, - normalizeStopSequences(stop), - ); - const outputText = - stopIndex === null ? rawOutputText : rawOutputText.slice(0, stopIndex); - const reasoningText = extractResponsesReasoningText(output); - const incompleteReason = - payload.incomplete_details && typeof payload.incomplete_details === 'object' - ? (payload.incomplete_details as { reason?: unknown }).reason - : undefined; - const finishReason = - payload.status === 'incomplete' - ? incompleteReason === 'content_filter' - ? 'content_filter' - : 'length' - : toolCalls.length - ? 'tool_calls' - : 'stop'; - - return { - choices: [ - { - finish_reason: finishReason, - index: 0, - message: { - content: outputText || null, - role: 'assistant', - ...(reasoningText ? { reasoning_content: reasoningText } : {}), - ...(toolCalls.length ? { tool_calls: toolCalls } : {}), - }, - }, - ], - created: Number(payload.created_at ?? Math.floor(Date.now() / 1000)), - id: String(payload.id ?? `chatcmpl-${crypto.randomUUID()}`), - model, - object: 'chat.completion', - usage: { - completion_tokens: outputTokens, - prompt_tokens: inputTokens, - total_tokens: Number(usage?.total_tokens ?? inputTokens + outputTokens), - }, - }; -}; - -const mapResponsesStreamToChat = ( - upstreamResponse: Response, - model: string, - proxyContext: ProxyContext, - route: string, - stop: string | string[] | undefined, - includeUsage: boolean, -): Response => { - const closer = createStreamCloser(); - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const responseId = `chatcmpl-${crypto.randomUUID()}`; - let reader: ReadableStreamDefaultReader | null = - upstreamResponse.body?.getReader() ?? null; - const fallbackUsage = parseUsageHeader(upstreamResponse); - let buffer = ''; - let emittedFinish = false; - let emittedUsage = false; - let hasToolCalls = false; - let latestUsage = fallbackUsage; - let usageRecorded = false; - const stopSequences = normalizeStopSequences(stop); - let pendingStopText = ''; - const toolIndexes = new Map(); - const toolCallIds = new Map(); - const customToolCallIds = new Set(); - const closedCustomToolCallIds = new Set(); - let nextToolIndex = 0; - let stoppedLocally = false; - - const getToolIndex = (itemId: string): number => { - const existing = toolIndexes.get(itemId); - if (existing !== undefined) return existing; - const index = nextToolIndex; - nextToolIndex += 1; - toolIndexes.set(itemId, index); - return index; - }; - - const encodeChunk = (choice: Record): Uint8Array => { - return encoder.encode( - `data: ${JSON.stringify({ - choices: [choice], - created: Math.floor(Date.now() / 1000), - id: responseId, - model, - object: 'chat.completion.chunk', - })}\n\n`, - ); - }; - - const enqueueUsage = ( - controller: ReadableStreamDefaultController, - ): void => { - if (!includeUsage || emittedUsage) return; - const usage = mapResponsesUsageToChat(latestUsage); - if (!usage) return; - - emittedUsage = true; - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ - choices: [], - created: Math.floor(Date.now() / 1000), - id: responseId, - model, - object: 'chat.completion.chunk', - usage, - })}\n\n`, - ), - ); - }; - - const recordStreamUsage = async (): Promise => { - if (usageRecorded) return; - usageRecorded = true; - try { - await recordProxyUsage({ - model, - proxyContext, - route, - usage: latestUsage, - }); - } catch (error) { - console.error('[CodeBuddy2API] Failed to record Responses stream usage', { - error, - route, - }); - } - }; - - const cancelAndReleaseReader = async (reason?: unknown): Promise => { - try { - await reader?.cancel(reason); - } catch (error) { - console.error('[CodeBuddy2API] Failed to cancel Responses stream', { - error, - route, - }); - } finally { - reader?.releaseLock(); - reader = null; - } - }; - - const emitCustomToolCallClosures = ( - controller: ReadableStreamDefaultController, - ): void => { - customToolCallIds.forEach((itemId) => { - if (closedCustomToolCallIds.has(itemId)) return; - const index = getToolIndex(itemId); - const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; - controller.enqueue( - encodeChunk({ - delta: { - tool_calls: [{ function: { arguments: '"}' }, id: callId, index }], - }, - index: 0, - }), - ); - closedCustomToolCallIds.add(itemId); - }); - }; - - const stream = new ReadableStream({ - async pull(controller) { - if (!reader) { - await recordStreamUsage(); - controller.close(); - return; - } - while (true) { - let readResult: ReadableStreamReadResult; - try { - readResult = await reader.read(); - } catch (error) { - await recordStreamUsage(); - reader.releaseLock(); - reader = null; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage !== null) { - closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); - return; - } - - controller.error(error); - return; - } - const { done, value } = readResult; - if (done) { - if (pendingStopText) { - controller.enqueue( - encodeChunk({ - delta: { content: pendingStopText }, - index: 0, - }), - ); - pendingStopText = ''; - } - emitCustomToolCallClosures(controller); - if (!emittedFinish) { - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: hasToolCalls ? 'tool_calls' : 'stop', - index: 0, - }), - ); - } - enqueueUsage(controller); - controller.enqueue(encodeDoneFrame()); - await recordStreamUsage(); - reader.releaseLock(); - reader = null; - controller.close(); - return; - } - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split(/\r?\n\r?\n/); - buffer = frames.pop() ?? ''; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - controller.enqueue(encodeDoneFrame()); - await cancelAndReleaseReader(); - await recordStreamUsage(); - controller.close(); - return; - } - let emitted = false; - for (const frame of frames) { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - controller.enqueue(encodeDoneFrame()); - await cancelAndReleaseReader(); - await recordStreamUsage(); - controller.close(); - return; - } - const dataLine = frame - .split(/\r?\n/) - .find((line) => line.startsWith('data: ')); - if (!dataLine || dataLine === 'data: [DONE]') continue; - // The upstream here is the chat pipeline, which reports a deadline as - // a terminal error chunk and closes cleanly. Without this the failure - // would be reported to the client as an empty successful response. - const upstreamError = readTimeoutFrame(frame); - - if (upstreamError !== null) { - closer.fail(controller, chatStreamErrorChunks(upstreamError)); - await cancelAndReleaseReader(); - await recordStreamUsage(); - return; - } - try { - const event = JSON.parse(dataLine.slice(6)) as { - delta?: unknown; - item?: unknown; - item_id?: unknown; - output_index?: unknown; - error?: unknown; - response?: unknown; - type?: unknown; - }; - latestUsage = extractResponsesUsage(event) ?? latestUsage; - if ( - stoppedLocally && - event.type !== 'response.completed' && - event.type !== 'response.incomplete' - ) { - continue; - } - if (event.type === 'response.output_text.delta') { - const delta = String(event.delta ?? ''); - if (stopSequences.length) { - pendingStopText += delta; - const stopIndex = findFirstStopSequence( - pendingStopText, - stopSequences, - ); - if (stopIndex !== null) { - const content = pendingStopText.slice(0, stopIndex); - if (content) { - controller.enqueue( - encodeChunk({ delta: { content }, index: 0 }), - ); - } - pendingStopText = ''; - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: 'stop', - index: 0, - }), - ); - emittedFinish = true; - stoppedLocally = true; - emitted = true; - continue; - } - - const pendingLength = getPendingStopPrefixLength( - pendingStopText, - stopSequences, - ); - const content = pendingStopText.slice( - 0, - pendingStopText.length - pendingLength, - ); - pendingStopText = pendingLength - ? pendingStopText.slice(-pendingLength) - : ''; - if (!content) continue; - controller.enqueue( - encodeChunk({ delta: { content }, index: 0 }), - ); - emitted = true; - continue; - } - controller.enqueue( - encodeChunk({ - delta: { content: delta }, - index: 0, - }), - ); - emitted = true; - continue; - } - if ( - event.type === 'response.reasoning_summary_text.delta' || - event.type === 'response.reasoning_text.delta' - ) { - controller.enqueue( - encodeChunk({ - delta: { reasoning_content: String(event.delta ?? '') }, - index: 0, - }), - ); - emitted = true; - continue; - } - if ( - event.type === 'response.output_item.added' && - event.item && - typeof event.item === 'object' - ) { - const item = event.item as { - arguments?: unknown; - call_id?: unknown; - id?: unknown; - input?: unknown; - name?: unknown; - type?: unknown; - }; - if ( - item.type !== 'function_call' && - item.type !== 'mcp_call' && - item.type !== 'custom_tool_call' - ) { - continue; - } - const isCustomToolCall = item.type === 'custom_tool_call'; - const initialArguments = isCustomToolCall - ? `{"input":"${JSON.stringify( - String(item.input ?? item.arguments ?? ''), - ).slice(1, -1)}` - : String(item.arguments ?? ''); - const itemId = String(item.id ?? item.call_id ?? nextToolIndex); - const index = getToolIndex(itemId); - const callId = String(item.call_id ?? item.id ?? itemId); - toolCallIds.set(itemId, callId); - if (isCustomToolCall) { - customToolCallIds.add(itemId); - } - hasToolCalls = true; - controller.enqueue( - encodeChunk({ - delta: { - tool_calls: [ - { - function: { - arguments: initialArguments, - name: String(item.name ?? 'function'), - }, - id: callId, - index, - type: 'function', - }, - ], - }, - index: 0, - }), - ); - emitted = true; - continue; - } - if ( - event.type === 'response.function_call_arguments.delta' || - event.type === 'response.mcp_call_arguments.delta' || - event.type === 'response.custom_tool_call_input.delta' - ) { - const itemId = String( - event.item_id ?? event.output_index ?? nextToolIndex, - ); - const index = getToolIndex(itemId); - const callId = toolCallIds.get(itemId) ?? `call_${index + 1}`; - toolCallIds.set(itemId, callId); - hasToolCalls = true; - const argumentDelta = - event.type === 'response.custom_tool_call_input.delta' - ? JSON.stringify(String(event.delta ?? '')).slice(1, -1) - : String(event.delta ?? ''); - controller.enqueue( - encodeChunk({ - delta: { - tool_calls: [ - { - function: { arguments: argumentDelta }, - id: callId, - index, - }, - ], - }, - index: 0, - }), - ); - emitted = true; - continue; - } - if (event.type === 'response.completed') { - if (pendingStopText) { - controller.enqueue( - encodeChunk({ - delta: { content: pendingStopText }, - index: 0, - }), - ); - pendingStopText = ''; - } - emitCustomToolCallClosures(controller); - if (!emittedFinish) { - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: hasToolCalls ? 'tool_calls' : 'stop', - index: 0, - }), - ); - } - enqueueUsage(controller); - emittedFinish = true; - emitted = true; - if (stoppedLocally) { - controller.enqueue(encodeDoneFrame()); - await cancelAndReleaseReader('Stop sequence matched'); - await recordStreamUsage(); - controller.close(); - return; - } - continue; - } - if (event.type === 'response.incomplete') { - if (pendingStopText) { - controller.enqueue( - encodeChunk({ - delta: { content: pendingStopText }, - index: 0, - }), - ); - pendingStopText = ''; - } - emitCustomToolCallClosures(controller); - const incompleteReason = - event.response && typeof event.response === 'object' - ? ( - (event.response as { incomplete_details?: unknown }) - .incomplete_details as { reason?: unknown } | undefined - )?.reason - : undefined; - if (!emittedFinish) { - controller.enqueue( - encodeChunk({ - delta: {}, - finish_reason: - incompleteReason === 'content_filter' - ? 'content_filter' - : 'length', - index: 0, - }), - ); - } - enqueueUsage(controller); - emittedFinish = true; - emitted = true; - if (stoppedLocally) { - controller.enqueue(encodeDoneFrame()); - await cancelAndReleaseReader('Stop sequence matched'); - await recordStreamUsage(); - controller.close(); - return; - } - continue; - } - if ( - event.type === 'response.failed' || - event.type === 'response.error' || - event.type === 'error' - ) { - const failure = - event.error ?? - (event.response && typeof event.response === 'object' - ? (event.response as { error?: unknown }).error - : undefined); - const message = - failure && typeof failure === 'object' - ? String( - (failure as { message?: unknown }).message ?? failure, - ) - : String(failure ?? 'Upstream Responses stream failed'); - controller.enqueue( - encoder.encode( - `data: ${JSON.stringify({ error: { message } })}\n\n`, - ), - ); - controller.enqueue(encodeDoneFrame()); - await cancelAndReleaseReader(); - await recordStreamUsage(); - controller.close(); - return; - } - } catch { - // Ignore malformed upstream events and continue reading. - } - } - if (stoppedLocally) continue; - if (emitted) return; - } - }, - async cancel(reason) { - await cancelAndReleaseReader(reason); - await recordStreamUsage(); - }, - }); - - return createSseResponse(stream, { - headers: CORS_HEADERS, - status: upstreamResponse.status, - }); -}; - -const aggregateToolCalls = ( - toolCalls: NonNullable, -): Array<{ - id?: string; - type?: string; - function: { - arguments: string; - name: string; - }; -}> => { - const aggregated = new Map< - string, - { - order: number; - id?: string; - type?: string; - function: { - arguments: string; - name: string; - }; - } - >(); - const latestKeyByIndex = new Map(); - - toolCalls.forEach((toolCall, position) => { - const normalizedId = createNormalizedToolCallId(toolCall.id, position); - const key = - (toolCall.id ? `id:${normalizedId}` : undefined) ?? - (typeof toolCall.index === 'number' - ? latestKeyByIndex.get(toolCall.index) - : undefined) ?? - `position:${position}`; - const current = aggregated.get(key) ?? { - order: aggregated.size, - function: { - arguments: '', - name: '', - }, - }; - - if (toolCall.id) { - current.id = normalizedId; - } - - if (toolCall.type) { - current.type = toolCall.type; - } - - if (toolCall.function?.name) { - current.function.name += toolCall.function.name; - } - - if (toolCall.function?.arguments) { - current.function.arguments += toolCall.function.arguments; - } - - aggregated.set(key, current); - - if (typeof toolCall.index === 'number') { - latestKeyByIndex.set(toolCall.index, key); - } - }); - - return [...aggregated.values()] - .sort((left, right) => left.order - right.order) - .map(({ order: _order, ...value }, index) => ({ - ...value, - id: value.id ?? createNormalizedToolCallId(undefined, index), - })); -}; - -const getToolCallStateKey = ( - toolCall: ToolCallChunk, - position: number, -): string => { - if (toolCall.id) { - return `id:${toolCall.id}`; - } - - if (typeof toolCall.index === 'number') { - return `index:${toolCall.index}`; - } - - return `position:${position}`; -}; - -const createNormalizedToolCallId = ( - sourceId: string | undefined, - normalizedIndex: number, -): string => { - if (sourceId && !sourceId.startsWith('tooluse_')) { - return sourceId; - } - - const suffix = - sourceId?.replace(/^tooluse_/, '') ?? - `${normalizedIndex}_${crypto.randomUUID().replaceAll('-', '')}`; - - return `call_${suffix}`; -}; - -const resolveToolCallMapping = ( - state: ToolCallNormalizationState, - toolCall: ToolCallChunk, - position: number, -): ToolCallMapping => { - const keys = toolCall.id - ? [`id:${toolCall.id}`] - : [ - typeof toolCall.index === 'number' ? `index:${toolCall.index}` : null, - `position:${position}`, - ].filter((value): value is string => value !== null); - const existing = keys - .map((key) => state.mappings.get(key)) - .find((value) => value !== undefined); - - if (existing) { - return existing; - } - - return { - id: createNormalizedToolCallId(toolCall.id, state.nextIndex), - index: state.nextIndex++, - }; -}; - -const normalizeStreamToolCalls = ( - chunk: ChatStreamChunk, - state: ToolCallNormalizationState, -): ChatStreamChunk => { - if (!chunk.choices?.length) { - return chunk; - } - - return { - ...chunk, - choices: chunk.choices.map((choice) => { - if (!choice.delta?.tool_calls?.length) { - return choice; - } - - return { - ...choice, - delta: { - ...choice.delta, - tool_calls: choice.delta.tool_calls.map((toolCall, position) => { - const mapping = resolveToolCallMapping(state, toolCall, position); - const sourceKey = getToolCallStateKey(toolCall, position); - - state.mappings.set(sourceKey, mapping); - - if (toolCall.id) { - state.mappings.set(`id:${toolCall.id}`, mapping); - } - - if (typeof toolCall.index === 'number') { - state.mappings.set(`index:${toolCall.index}`, mapping); - } - - return { - ...toolCall, - id: mapping.id, - index: mapping.index, - }; - }), - }, - }; - }), - }; -}; - -const normalizeStreamingResponse = ({ - model, - proxyContext, - route, - upstreamResponse, -}: { - model: string; - proxyContext: ProxyContext; - route: string; - upstreamResponse: Response; -}): Response => { - if (!upstreamResponse.body) { - return createSseResponse(null, { - headers: CORS_HEADERS, - status: upstreamResponse.status, - }); - } - - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - const state: ToolCallNormalizationState = { - mappings: new Map(), - nextIndex: 0, - }; - let reader: ReadableStreamDefaultReader | null = null; - let cancelled = false; - const closer = createStreamCloser(); - const releaseReader = (): void => { - reader?.releaseLock(); - reader = null; - }; - - const stream = new ReadableStream({ - start: (controller) => { - const upstreamReader = upstreamResponse.body!.getReader(); - reader = upstreamReader; - let buffer = ''; - let latestUsage: unknown = null; - - const processFrame = (frame: string): string => { - const lines = frame.split('\n'); - const lineIndex = lines.findIndex((line) => line.startsWith('data: ')); - - if (lineIndex === -1) { - return frame; - } - - const raw = lines[lineIndex]?.slice(6).trim() ?? ''; - - if (!raw || raw === '[DONE]') { - return frame; - } - - try { - const chunk = JSON.parse(raw) as ChatStreamChunk; - if (chunk.usage !== undefined) { - latestUsage = chunk.usage; - } - const normalized = normalizeStreamToolCalls(chunk, state); - lines[lineIndex] = `data: ${JSON.stringify(normalized)}`; - return lines.join('\n'); - } catch { - return frame; - } - }; - - const flushFrames = (frames: string[]): void => { - frames.forEach((frame) => { - if (frame.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - return; - } - controller.enqueue(encoder.encode(`${processFrame(frame)}\n\n`)); - }); - }; - - const pump = async (): Promise => { - while (true) { - const { done, value } = await upstreamReader.read(); - - if (cancelled) { - return; - } - - if (done) { - if (buffer.trim()) { - flushFrames([buffer]); - } - - await recordProxyUsage({ - model, - proxyContext, - route, - usage: latestUsage, - }); - - releaseReader(); - try { - controller.close(); - } catch { - // The downstream stream may have been cancelled while the pump was completing. - } - return; - } - - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop()!; - if (buffer.length > MAX_STREAM_FRAME_LENGTH) { - controller.enqueue( - encoder.encode( - 'data: {"error":{"message":"Upstream SSE frame exceeds the maximum size"}}\n\n', - ), - ); - try { - await reader!.cancel(); - } finally { - releaseReader(); - controller.close(); - } - return; - } - flushFrames(frames); - } - }; - - void pump().catch((error) => { - if (cancelled) return; - const timeoutMessage = toUpstreamTimeoutMessage(error); - - if (timeoutMessage !== null) { - closer.fail(controller, chatStreamErrorChunks(timeoutMessage)); - return; - } - - controller.error(error); - }); - }, - async cancel(reason): Promise { - cancelled = true; - closer.mark(); - try { - await reader?.cancel(reason); - } finally { - releaseReader(); - } - }, - }); - - return createSseResponse(stream, { - headers: CORS_HEADERS, - status: upstreamResponse.status, - }); -}; - -const aggregateUpstreamStream = async ( - upstreamResponse: Response, - fallbackModel: string, -): Promise<{ model: string; response: Response; usage: unknown }> => { - const payloadText = await upstreamResponse.text(); - const toolCalls: NonNullable = []; - let responseId = ''; - let responseObject = 'chat.completion'; - let created = Math.floor(Date.now() / 1000); - let model = fallbackModel; - let content = ''; - let reasoningContent = ''; - let finishReason: string | null = 'stop'; - let role = 'assistant'; - let usage: unknown = null; - - for (const frame of payloadText.split('\n\n')) { - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - continue; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - continue; - } - - let chunk: ChatStreamChunk; - - try { - chunk = JSON.parse(raw) as ChatStreamChunk; - } catch { - return { - model: fallbackModel, - response: createErrorResponse( - 502, - 'Failed to parse upstream SSE frame', - ), - usage: null, - }; - } - - if (chunk.id) { - responseId = chunk.id; - } - - if (chunk.object) { - responseObject = chunk.object.replace(/\.chunk$/, ''); - } - - if (typeof chunk.created === 'number') { - created = chunk.created; - } - - if (chunk.model) { - model = chunk.model; - } - - if (chunk.usage !== undefined) { - usage = chunk.usage; - } - - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - - if (delta?.role) { - role = delta.role; - } - - if (delta?.content) { - content += delta.content; - } - - if (delta?.reasoning_content ?? delta?.reasoning) { - reasoningContent += delta.reasoning_content ?? delta.reasoning; - } - - if (delta?.tool_calls?.length) { - toolCalls.push(...delta.tool_calls); - } - - if (choice?.finish_reason !== undefined) { - finishReason = choice.finish_reason ?? finishReason; - } - } - - const aggregatedToolCalls = aggregateToolCalls(toolCalls); - const message: Record = { - role, - content: content || null, - }; - - if (reasoningContent) { - message.reasoning_content = reasoningContent; - } - - if (aggregatedToolCalls.length) { - message.tool_calls = aggregatedToolCalls; - } - - return { - model, - response: Response.json({ - id: responseId || `chatcmpl_${crypto.randomUUID().replaceAll('-', '')}`, - object: responseObject, - created, - model, - choices: [ - { - index: 0, - message, - finish_reason: - finishReason ?? - (aggregatedToolCalls.length ? 'tool_calls' : 'stop'), - }, - ], - usage, - }), - usage, - }; -}; - -const isServerWebToolName = (name: string): boolean => { - const normalized = normalizeToolName(name); - - return ( - normalized === normalizeToolName(WEB_SEARCH_TOOL_NAME) || - normalized === normalizeToolName(WEB_FETCH_TOOL_NAME) - ); -}; - -const SERVER_WEB_TOOL_NAMES = [ - normalizeToolName(WEB_SEARCH_TOOL_NAME), - normalizeToolName(WEB_FETCH_TOOL_NAME), -]; - -interface StreamProbeState { - toolNames: Map; -} - -const mergeToolName = (previous: string, incoming: string): string => { - if (!previous || incoming.startsWith(previous)) { - return incoming; - } - - if (!incoming || previous.endsWith(incoming)) { - return previous; - } - - return previous + incoming; -}; - -const classifyStreamFrame = ( - frame: string, - state: StreamProbeState, - serverToolNames: string[], -): 'passthrough' | 'server-tool' | null => { - const line = frame - .split('\n') - .find((segment) => segment.startsWith('data: ')); - - if (!line) { - return null; - } - - const raw = line.slice(6).trim(); - - if (!raw || raw === '[DONE]') { - return raw === '[DONE]' ? 'passthrough' : null; - } - - try { - const chunk = JSON.parse(raw) as ChatStreamChunk; - - for (const choice of chunk.choices ?? []) { - const delta = choice.delta; - const toolCalls = delta?.tool_calls ?? []; - let hasNonServerTool = false; - - for (const [position, toolCall] of toolCalls.entries()) { - const incoming = toolCall.function?.name; - - if (typeof incoming !== 'string' || !incoming) { - continue; - } - - const key = - typeof toolCall.index === 'number' - ? `index:${toolCall.index}` - : toolCall.id - ? `id:${toolCall.id}` - : `position:${position}`; - const name = mergeToolName(state.toolNames.get(key) ?? '', incoming); - const normalized = normalizeToolName(name); - - state.toolNames.set(key, name); - - if (isServerWebToolName(name) && serverToolNames.includes(normalized)) { - return 'server-tool'; - } - - if ( - normalized && - !serverToolNames.some((serverName) => - serverName.startsWith(normalized), - ) - ) { - hasNonServerTool = true; - } - } - - if ( - delta?.content || - delta?.reasoning_content || - delta?.reasoning || - hasNonServerTool || - choice.finish_reason != null - ) { - return 'passthrough'; - } - } - } catch { - return null; - } - - return null; -}; - -const concatenateChunks = (chunks: Uint8Array[]): ArrayBuffer => { - const size = chunks.reduce((total, chunk) => total + chunk.byteLength, 0); - const combined = new Uint8Array(size); - let offset = 0; - - for (const chunk of chunks) { - combined.set(chunk, offset); - offset += chunk.byteLength; - } - - return combined.buffer; -}; - -const createResponseWithBody = (body: BodyInit, response: Response): Response => - new Response(body, { - headers: response.headers, - status: response.status, - statusText: response.statusText, - }); - -const createReplayStreamResponse = ({ - chunks, - reader, - response, -}: { - chunks: Uint8Array[]; - reader: ReadableStreamDefaultReader; - response: Response; -}): Response => { - let cancelled = false; - - const stream = new ReadableStream({ - start: (controller) => { - for (const chunk of chunks) { - controller.enqueue(chunk); - } - - const pump = async (): Promise => { - while (true) { - const { done, value } = await reader.read(); - - if (cancelled) { - return; - } - - if (done) { - reader.releaseLock(); - controller.close(); - return; - } - - controller.enqueue(value); - } - }; - - void pump().catch((error) => { - if (!cancelled) { - controller.error(error); - } - }); - }, - async cancel(reason): Promise { - cancelled = true; - try { - await reader.cancel(reason); - } finally { - reader.releaseLock(); - } - }, - }); - - return createResponseWithBody(stream, response); -}; - -const detectServerToolStream = async ( - response: Response, - fallbackModel: string, - serverToolNames: string[], -): Promise => { - if (!response.body) { - return response; - } - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - const chunks: Uint8Array[] = []; - const state: StreamProbeState = { toolNames: new Map() }; - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - - if (done) { - reader.releaseLock(); - return createResponseWithBody(concatenateChunks(chunks), response); - } - - chunks.push(value); - buffer += decoder.decode(value, { stream: true }); - const frames = buffer.split('\n\n'); - buffer = frames.pop() ?? ''; - - for (const frame of frames) { - const classification = classifyStreamFrame(frame, state, serverToolNames); - - if (classification === 'passthrough') { - return createReplayStreamResponse({ chunks, reader, response }); - } - - if (classification === 'server-tool') { - while (true) { - const remainder = await reader.read(); - - if (remainder.done) { - reader.releaseLock(); - break; - } - - chunks.push(remainder.value); - } - - return ( - await aggregateUpstreamStream( - new Response(concatenateChunks(chunks)), - fallbackModel, - ) - ).response; - } - } - } -}; - -export const getModelsForCredential = async ({ - bearerToken, - credentialData, -}: { - bearerToken: string; - credentialData: CredentialData; -}): Promise => { - const configuredEndpoint = await getCodeBuddyApiEndpoint(); - const headers = new Headers({ - Accept: 'application/json', - Authorization: `Bearer ${bearerToken}`, - 'X-Product': 'SaaS', - }); - const domain = getCredentialValue(credentialData, ['domain']); - const apiEndpoint = String(domain ?? '') - .toLowerCase() - .endsWith('workbuddy.ai') - ? 'https://www.workbuddy.ai' - : configuredEndpoint; - const enterpriseId = getCredentialValue(credentialData, [ - 'enterprise_id', - 'enterpriseId', - ]); - const tenantId = - getCredentialValue(credentialData, ['tenant_id', 'tenantId']) ?? - enterpriseId; - const userId = getCredentialValue(credentialData, ['user_id', 'userId']); - - if (domain) { - headers.set('X-Domain', String(domain)); - } - - if (enterpriseId) { - headers.set('X-Enterprise-Id', String(enterpriseId)); - } - - if (tenantId) { - headers.set('X-Tenant-Id', String(tenantId)); - } - if (userId) { - headers.set('X-User-Id', String(userId)); - } - - const fetchModels = async (path: string): Promise => - fetch(new URL(path, apiEndpoint), { - headers, - signal: AbortSignal.timeout(15_000), - }); - let response = await fetchModels('/v3/config'); - - if ([400, 404, 405].includes(response.status)) { - // Upstream splits this route by account scope: enterprise accounts must hit - // their own segment, otherwise they are served the personal model catalog. - const enterpriseScope = String(enterpriseId ?? '').trim() || 'personal'; - response = await fetchModels( - `/console/enterprises/${encodeURIComponent(enterpriseScope)}/models`, - ); - } - - if (!response.ok) { - throw new Error(`Model discovery failed with status ${response.status}`); - } - - const payload = (await response.json()) as { - code?: unknown; - data?: { - agents?: Array<{ models?: unknown; name?: unknown }>; - models?: Array<{ disabled?: unknown; id?: unknown; name?: unknown }>; - }; - }; - - if (payload.code !== 0) { - throw new Error('Model discovery returned an unsuccessful response'); - } - - const cliModels = payload.data?.agents?.find( - (agent) => agent.name === 'cli', - )?.models; - const modelsById = new Map( - (payload.data?.models ?? []).flatMap((model) => { - const id = typeof model.id === 'string' ? model.id.trim() : ''; - - if (!id || model.disabled === true) { - return []; - } - - return [ - [ - id, - { - displayName: - typeof model.name === 'string' && model.name.trim() - ? model.name - : id, - id, - }, - ] as const, - ]; - }), - ); - const declaredModelIds = new Set( - (payload.data?.models ?? []) - .map((model) => (typeof model.id === 'string' ? model.id.trim() : '')) - .filter(Boolean), - ); - - if (!Array.isArray(cliModels)) { - return []; - } - - return cliModels.flatMap((modelId) => { - if (typeof modelId !== 'string') { - return []; - } - - const model = modelsById.get(modelId); - if (!model && declaredModelIds.has(modelId)) { - return []; - } - return [ - model ?? { - displayName: modelId, - id: modelId, - }, - ]; - }); -}; - -export const getModelsForCredentials = async ( - credentials: CredentialRecord[], -): Promise => { - const settled = await Promise.allSettled( - credentials.map((credential) => { - const supportedModels = getCredentialSupportedModels(credential.data); - - if (supportedModels.length) { - return Promise.resolve( - supportedModels.map((id) => ({ displayName: id, id })), - ); - } - - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - return bearerToken - ? getModelsForCredential({ - bearerToken, - credentialData: credential.data, - }) - : Promise.resolve([]); - }), - ); - const models = new Map(); - - settled.forEach((result) => { - if (result.status !== 'fulfilled') { - return; - } - - result.value.forEach((model) => { - models.set(model.id, model); - }); - }); - - return [...models.values()].sort((left, right) => - left.id.localeCompare(right.id), - ); -}; - -export const getModelsByCredential = async ( - credentials: CredentialRecord[], -): Promise< - Record -> => { - const results = await Promise.all( - credentials.map(async (credential) => { - const bearerToken = String( - credential.data.bearer_token ?? credential.data.access_token ?? '', - ).trim(); - - try { - const models = bearerToken - ? await getModelsForCredential({ - bearerToken, - credentialData: credential.data, - }) - : []; - - return [credential.filename, { error: null, models }] as const; - } catch (error) { - return [ - credential.filename, - { - error: - error instanceof Error ? error.message : 'Model discovery failed', - models: [], - }, - ] as const; - } - }), - ); - - return Object.fromEntries(results); -}; - -export const getModelsResponse = async ( - request?: NextRequest, -): Promise => { - const accessKey = request ? await resolveRequestAccessKey(request) : null; - const models = ( - await getModelsForCredentials( - await listEligibleCredentialRecords(accessKey?.credentialFilenames), - ) - ).map((model) => ({ - id: model.id, - slug: model.id, - display_name: model.displayName, - object: 'model', - created: 0, - owned_by: 'codebuddy', - })); - - return Response.json({ - object: 'list', - data: models, - models, - }); -}; + aggregateUpstreamStream, + normalizeStreamingResponse, +} from './codebuddy-chat-stream'; +import { resolveProxyContext } from './codebuddy-context'; +import { + buildResponsesBodyFromChat, + getUnsupportedResponsesChatOptions, + normalizeResponsesUpstreamBody, +} from './codebuddy-responses-request'; +import { + mapResponsesPayloadToChat, + mapResponsesStreamToChat, +} from './codebuddy-responses-response'; +import { detectServerToolStream } from './codebuddy-server-tools'; +import { + SERVER_WEB_TOOL_NAMES, + type ChatRequestBody, + type ProxyContext, +} from './codebuddy-types'; +import { + extractResponsesId, + extractResponsesUsage, + logUpstreamFailure, + parseUsageHeader, + recordProxyUsage, +} from './codebuddy-usage'; +import { trackResponsesUsageStream } from './codebuddy-usage-stream'; +import { + buildUpstreamBody, + buildUpstreamHeaders, + headersToRecord, +} from './codebuddy-upstream'; +import { + enqueueUpstreamResponseSnapshot, + setDebugTraceCredential, + setDebugTraceError, + setDebugUpstreamRequest, + type DebugTrace, +} from '../domain/debug'; +import { + getApiFirstDeltaTimeoutMs, + getCodeBuddyApiEndpoint, + getDefaultModel, +} from '../domain/config'; +import { withCodeBuddyToken } from '../search/token'; +import { + normalizeToolName, + WEB_FETCH_TOOL_NAME, + WEB_SEARCH_TOOL_NAME, +} from '../search/tool'; +import { createErrorResponse } from '../shared/http'; +import { fetchWithDeadline } from '../shared/upstream-timeout'; +import { + attachServerToolExecutions, + attachServerToolTurns, + type ChatCompletionPayload, + executeWebSearchLoop, + type ServerToolCallbacks, + synthesizeChatCompletionStream, +} from './web-search-loop'; /** * One round trip to `/v2/chat/completions`. Split out from @@ -3491,3 +664,27 @@ export const proxyResponsesUpstream = async ( ); } }; + +// Re-exported for the importers that reached these through this module before +// the split: the route handlers, the domain layer, and the test suite. +export type { + ChatRequestBody, + DiscoveredModel, + ProxyContext, +} from './codebuddy-types'; +export { + createProxyContextFromCredential, + resolveProxyContext, + resolveProxyContextByCredentialFilename, +} from './codebuddy-context'; +export { buildUpstreamHeaders } from './codebuddy-upstream'; +export { + extractImageUrl, + isImageContentPart, +} from './codebuddy-responses-request'; +export { + getModelsByCredential, + getModelsForCredential, + getModelsForCredentials, + getModelsResponse, +} from './codebuddy-models'; From ff6f6a8de7e3862547af65a3ce472e72a41ec87e Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 22:00:54 +0800 Subject: [PATCH 6/7] test(shared): cover the extracted content and SSE helpers The shared modules were assembled from code that lived inside files whose overall coverage hid which branches were untested. Now that they stand alone, the gaps are visible, so cover them directly. --- tests/server/shared-content.test.ts | 74 +++++++++++++++++++++++++++++ tests/server/shared-sse.test.ts | 69 +++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 tests/server/shared-content.test.ts create mode 100644 tests/server/shared-sse.test.ts diff --git a/tests/server/shared-content.test.ts b/tests/server/shared-content.test.ts new file mode 100644 index 0000000..db7b063 --- /dev/null +++ b/tests/server/shared-content.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; + +import { + asRecord, + readReasoning, + stringifyContent, +} from '@/lib/server/shared/content'; + +describe('asRecord', () => { + it('accepts a plain object', () => { + expect(asRecord({ a: 1 })).toEqual({ a: 1 }); + }); + + it('rejects everything that is not a plain object', () => { + expect(asRecord(null)).toBeNull(); + expect(asRecord(undefined)).toBeNull(); + expect(asRecord('text')).toBeNull(); + expect(asRecord(0)).toBeNull(); + expect(asRecord([])).toBeNull(); + expect(asRecord([{ a: 1 }])).toBeNull(); + }); +}); + +describe('stringifyContent', () => { + it('passes a string through untouched', () => { + expect(stringifyContent('hello')).toBe('hello'); + expect(stringifyContent('')).toBe(''); + }); + + it('reports nothing for an absent value', () => { + expect(stringifyContent(undefined)).toBe(''); + expect(stringifyContent(null)).toBe(''); + }); + + it('concatenates the text of each part of an array', () => { + expect(stringifyContent(['a', { text: 'b' }, 'c'])).toBe('abc'); + }); + + it('falls back to JSON for a part without text', () => { + expect(stringifyContent([{ url: 'https://example.com/x.png' }])).toBe( + '{"url":"https://example.com/x.png"}', + ); + }); + + it('renders an empty text part as nothing', () => { + expect(stringifyContent([{ text: undefined }])).toBe(''); + }); + + it('JSON-encodes a value of any other shape', () => { + expect(stringifyContent({ a: 1 })).toBe('{"a":1}'); + expect(stringifyContent(42)).toBe('42'); + }); +}); + +describe('readReasoning', () => { + it('prefers reasoning_content over reasoning', () => { + expect(readReasoning({ reasoning: 'soon', reasoning_content: 'now' })).toBe( + 'now', + ); + }); + + it('falls back to reasoning', () => { + expect(readReasoning({ reasoning: 'text' })).toBe('text'); + }); + + it('reports nothing when the message carries neither', () => { + expect(readReasoning({})).toBe(''); + expect(readReasoning(undefined)).toBe(''); + }); + + it('ignores reasoning of an unexpected type', () => { + expect(readReasoning({ reasoning: 42 })).toBe(''); + }); +}); diff --git a/tests/server/shared-sse.test.ts b/tests/server/shared-sse.test.ts new file mode 100644 index 0000000..c177c6c --- /dev/null +++ b/tests/server/shared-sse.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest'; + +import { + createSseHeaders, + createSseResponse, + DONE_FRAME_TEXT, + encodeDoneFrame, + encodeEventFrame, + eventFrameText, +} from '@/lib/server/shared/sse'; + +const decoder = new TextDecoder(); + +const decode = (chunk: Uint8Array): string => decoder.decode(chunk); + +describe('createSseHeaders', () => { + it('carries the three headers every stream needs', () => { + expect(createSseHeaders()).toEqual({ + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + 'Content-Type': 'text/event-stream; charset=utf-8', + }); + }); + + it('lets a caller add to the set without losing the rest', () => { + const headers = createSseHeaders({ 'Access-Control-Allow-Origin': '*' }); + + expect(headers['Access-Control-Allow-Origin']).toBe('*'); + expect(headers['Cache-Control']).toBe('no-cache'); + }); +}); + +describe('createSseResponse', () => { + it('defaults to 200', async () => { + const response = createSseResponse('data: x\n\n'); + + expect(response.status).toBe(200); + expect(await response.text()).toBe('data: x\n\n'); + }); + + it('passes an explicit status through, including a null-body one', () => { + const response = createSseResponse(null, { status: 204 }); + + expect(response.status).toBe(204); + expect(response.body).toBeNull(); + }); +}); + +describe('frame serialisation', () => { + it('writes a named event with its type line', () => { + const frame = decode(encodeEventFrame('response.created', { id: 'r1' })); + + expect(frame).toBe('event: response.created\ndata: {"id":"r1"}\n\n'); + }); + + it('writes the terminating frame', () => { + expect(decode(encodeDoneFrame())).toBe('data: [DONE]\n\n'); + }); + + it('omits the terminator from the text forms, which are joined with it', () => { + expect(eventFrameText('response.created', { id: 'r1' })).toBe( + 'event: response.created\ndata: {"id":"r1"}', + ); + expect(DONE_FRAME_TEXT).toBe('data: [DONE]'); + expect([eventFrameText('a', {}), DONE_FRAME_TEXT, ''].join('\n\n')).toBe( + 'event: a\ndata: {}\n\ndata: [DONE]\n\n', + ); + }); +}); From 85de83e6a5e026bf8877275e41111d128f534624 Mon Sep 17 00:00:00 2001 From: orangeboyChen Date: Thu, 17 Sep 2026 22:18:37 +0800 Subject: [PATCH 7/7] refactor(proxy): nest the split modules in folders per protocol Grouping the modules extracted from each god file behind a folder -- proxy/codebuddy/, proxy/responses/, proxy/anthropic/ and proxy/server-tool/ -- and dropping the now-redundant filename prefixes, so codebuddy/models.ts reads as "the models module of codebuddy" rather than as a sibling of codebuddy.ts. The entry points stay at the proxy root with their import paths intact: TypeScript and the bundler resolve ./codebuddy to codebuddy.ts, not to the folder of the same name. No behaviour change; the module graph is identical, cycles included. --- lib/server/proxy/anthropic.ts | 10 +++---- .../content.ts} | 4 +-- .../errors.ts} | 2 +- .../request.ts} | 10 +++---- .../response.ts} | 6 ++-- .../stream.ts} | 23 +++++++-------- .../types.ts} | 0 lib/server/proxy/codebuddy.ts | 28 +++++++++---------- .../chat-stream.ts} | 10 +++---- .../context.ts} | 8 +++--- .../models.ts} | 10 +++---- .../responses-request.ts} | 6 ++-- .../responses-response.ts} | 8 +++--- .../server-tools.ts} | 6 ++-- .../types.ts} | 2 +- .../upstream.ts} | 12 ++++---- .../usage-stream.ts} | 6 ++-- .../usage.ts} | 4 +-- lib/server/proxy/responses.ts | 16 +++++------ .../event-stream.ts} | 22 +++++++-------- .../{responses-ids.ts => responses/ids.ts} | 0 .../payload.ts} | 20 ++++++------- .../session.ts} | 9 ++---- .../stream.ts} | 22 +++++++-------- .../tools.ts} | 8 +++--- .../transcript.ts} | 14 +++++----- .../types.ts} | 0 .../args.ts} | 4 +-- .../classify.ts} | 8 +++--- .../execution.ts} | 12 ++++---- .../payload.ts} | 4 +-- .../sse.ts} | 4 +-- .../stream.ts} | 9 ++---- .../turns.ts} | 4 +-- .../types.ts} | 4 +-- lib/server/proxy/web-search-loop.ts | 22 +++++++-------- 36 files changed, 164 insertions(+), 173 deletions(-) rename lib/server/proxy/{anthropic-content.ts => anthropic/content.ts} (97%) rename lib/server/proxy/{anthropic-errors.ts => anthropic/errors.ts} (94%) rename lib/server/proxy/{anthropic-request.ts => anthropic/request.ts} (98%) rename lib/server/proxy/{anthropic-response.ts => anthropic/response.ts} (97%) rename lib/server/proxy/{anthropic-stream.ts => anthropic/stream.ts} (97%) rename lib/server/proxy/{anthropic-types.ts => anthropic/types.ts} (100%) rename lib/server/proxy/{codebuddy-chat-stream.ts => codebuddy/chat-stream.ts} (98%) rename lib/server/proxy/{codebuddy-context.ts => codebuddy/context.ts} (95%) rename lib/server/proxy/{codebuddy-models.ts => codebuddy/models.ts} (95%) rename lib/server/proxy/{codebuddy-responses-request.ts => codebuddy/responses-request.ts} (98%) rename lib/server/proxy/{codebuddy-responses-response.ts => codebuddy/responses-response.ts} (99%) rename lib/server/proxy/{codebuddy-server-tools.ts => codebuddy/server-tools.ts} (96%) rename lib/server/proxy/{codebuddy-types.ts => codebuddy/types.ts} (99%) rename lib/server/proxy/{codebuddy-upstream.ts => codebuddy/upstream.ts} (95%) rename lib/server/proxy/{codebuddy-usage-stream.ts => codebuddy/usage-stream.ts} (97%) rename lib/server/proxy/{codebuddy-usage.ts => codebuddy/usage.ts} (96%) rename lib/server/proxy/{responses-event-stream.ts => responses/event-stream.ts} (94%) rename lib/server/proxy/{responses-ids.ts => responses/ids.ts} (100%) rename lib/server/proxy/{responses-payload.ts => responses/payload.ts} (96%) rename lib/server/proxy/{responses-session.ts => responses/session.ts} (97%) rename lib/server/proxy/{responses-stream.ts => responses/stream.ts} (97%) rename lib/server/proxy/{responses-tools.ts => responses/tools.ts} (99%) rename lib/server/proxy/{responses-transcript.ts => responses/transcript.ts} (97%) rename lib/server/proxy/{responses-types.ts => responses/types.ts} (100%) rename lib/server/proxy/{server-tool-args.ts => server-tool/args.ts} (96%) rename lib/server/proxy/{server-tool-classify.ts => server-tool/classify.ts} (97%) rename lib/server/proxy/{server-tool-execution.ts => server-tool/execution.ts} (91%) rename lib/server/proxy/{server-tool-payload.ts => server-tool/payload.ts} (96%) rename lib/server/proxy/{server-tool-sse.ts => server-tool/sse.ts} (97%) rename lib/server/proxy/{server-tool-stream.ts => server-tool/stream.ts} (96%) rename lib/server/proxy/{server-tool-turns.ts => server-tool/turns.ts} (98%) rename lib/server/proxy/{server-tool-types.ts => server-tool/types.ts} (97%) diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index ffccfb7..5e26861 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -5,20 +5,20 @@ import { anthropicErrorType, createAnthropicError, getUpstreamErrorMessage, -} from './anthropic-errors'; +} from './anthropic/errors'; import { buildChatRequestBody, shouldBridgeAnthropicServerTools, -} from './anthropic-request'; -import { mapOpenAIResponseToAnthropic } from './anthropic-response'; +} from './anthropic/request'; +import { mapOpenAIResponseToAnthropic } from './anthropic/response'; import { createAnthropicServerToolEventStream, mapOpenAIStreamToAnthropicSSE, -} from './anthropic-stream'; +} from './anthropic/stream'; import type { AnthropicMessagesRequestBody, OpenAIChatResponse, -} from './anthropic-types'; +} from './anthropic/types'; import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; import { getServerToolExecutions, getServerToolTurns } from './web-search-loop'; diff --git a/lib/server/proxy/anthropic-content.ts b/lib/server/proxy/anthropic/content.ts similarity index 97% rename from lib/server/proxy/anthropic-content.ts rename to lib/server/proxy/anthropic/content.ts index aafcc48..88df7d8 100644 --- a/lib/server/proxy/anthropic-content.ts +++ b/lib/server/proxy/anthropic/content.ts @@ -1,4 +1,4 @@ -import { stringifyContent } from '../shared/content'; +import { stringifyContent } from '../../shared/content'; import type { AnthropicContentBlock, AnthropicImageSource, @@ -7,7 +7,7 @@ import type { ChatImageBlock, ChatTextBlock, ChatTextContent, -} from './anthropic-types'; +} from './types'; // --------------------------------------------------------------------------- // Helpers diff --git a/lib/server/proxy/anthropic-errors.ts b/lib/server/proxy/anthropic/errors.ts similarity index 94% rename from lib/server/proxy/anthropic-errors.ts rename to lib/server/proxy/anthropic/errors.ts index fce19df..1cd6a13 100644 --- a/lib/server/proxy/anthropic-errors.ts +++ b/lib/server/proxy/anthropic/errors.ts @@ -1,4 +1,4 @@ -import { extractErrorMessage } from '../shared/http'; +import { extractErrorMessage } from '../../shared/http'; export const anthropicErrorType = (status: number): string => status === 401 diff --git a/lib/server/proxy/anthropic-request.ts b/lib/server/proxy/anthropic/request.ts similarity index 98% rename from lib/server/proxy/anthropic-request.ts rename to lib/server/proxy/anthropic/request.ts index bce63f8..d4fb6ce 100644 --- a/lib/server/proxy/anthropic-request.ts +++ b/lib/server/proxy/anthropic/request.ts @@ -2,8 +2,8 @@ import { getDefaultModel, isWebFetchEnabled, isWebSearchEnabled, -} from '../domain/config'; -import { stringifyContent } from '../shared/content'; +} from '../../domain/config'; +import { stringifyContent } from '../../shared/content'; import { markServerTool, normalizeToolName, @@ -11,13 +11,13 @@ import { WEB_FETCH_TOOL_TYPE_PREFIX, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; +} from '../../search/tool'; import { buildChatImageUrl, createAnthropicId, extractSystemText, mapContentPartsToChat, -} from './anthropic-content'; +} from './content'; import type { AnthropicContentBlock, AnthropicMessage, @@ -26,7 +26,7 @@ import type { ChatContentPart, ChatImageBlock, ChatMessage, -} from './anthropic-types'; +} from './types'; // --------------------------------------------------------------------------- // Request translation: Anthropic → OpenAI diff --git a/lib/server/proxy/anthropic-response.ts b/lib/server/proxy/anthropic/response.ts similarity index 97% rename from lib/server/proxy/anthropic-response.ts rename to lib/server/proxy/anthropic/response.ts index dd0cca1..d4c57a3 100644 --- a/lib/server/proxy/anthropic-response.ts +++ b/lib/server/proxy/anthropic/response.ts @@ -1,10 +1,10 @@ -import { createAnthropicId } from './anthropic-content'; +import { createAnthropicId } from './content'; import type { AnthropicContentBlock, OpenAIChatResponse, OpenAIUsage, -} from './anthropic-types'; -import type { ServerToolExecution, ServerToolTurn } from './web-search-loop'; +} from './types'; +import type { ServerToolExecution, ServerToolTurn } from '../web-search-loop'; // --------------------------------------------------------------------------- // Response translation: OpenAI → Anthropic (non-streaming) diff --git a/lib/server/proxy/anthropic-stream.ts b/lib/server/proxy/anthropic/stream.ts similarity index 97% rename from lib/server/proxy/anthropic-stream.ts rename to lib/server/proxy/anthropic/stream.ts index 3f7363b..7e729fa 100644 --- a/lib/server/proxy/anthropic-stream.ts +++ b/lib/server/proxy/anthropic/stream.ts @@ -1,35 +1,32 @@ import type { NextRequest } from 'next/server'; -import type { DebugTrace } from '../domain/debug'; -import { createSseResponse } from '../shared/sse'; +import type { DebugTrace } from '../../domain/debug'; +import { createSseResponse } from '../../shared/sse'; import { anthropicStreamErrorChunks, createStreamCloser, toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { proxyChatCompletions, type ChatRequestBody } from './codebuddy'; -import { createAnthropicId } from './anthropic-content'; -import { - anthropicErrorType, - getUpstreamErrorMessage, -} from './anthropic-errors'; +} from '../../shared/upstream-timeout'; +import { proxyChatCompletions, type ChatRequestBody } from '../codebuddy'; +import { createAnthropicId } from './content'; +import { anthropicErrorType, getUpstreamErrorMessage } from './errors'; import { buildAnthropicServerToolBlocks, mapFinishReasonToAnthropic, mapOpenAIUsageToAnthropic, -} from './anthropic-response'; -import { MAX_STREAM_FRAME_LENGTH } from './anthropic-types'; +} from './response'; +import { MAX_STREAM_FRAME_LENGTH } from './types'; import type { OpenAIStreamChunk, OpenAIStreamError, OpenAIUsage, StreamingToolUseState, -} from './anthropic-types'; +} from './types'; import { getServerToolExecutions, getServerToolStreamEvent, type ServerToolExecution, -} from './web-search-loop'; +} from '../web-search-loop'; // --------------------------------------------------------------------------- // Response translation: OpenAI SSE → Anthropic SSE (streaming) diff --git a/lib/server/proxy/anthropic-types.ts b/lib/server/proxy/anthropic/types.ts similarity index 100% rename from lib/server/proxy/anthropic-types.ts rename to lib/server/proxy/anthropic/types.ts diff --git a/lib/server/proxy/codebuddy.ts b/lib/server/proxy/codebuddy.ts index d7dfdb3..02f3476 100644 --- a/lib/server/proxy/codebuddy.ts +++ b/lib/server/proxy/codebuddy.ts @@ -3,36 +3,36 @@ import type { NextRequest } from 'next/server'; import { aggregateUpstreamStream, normalizeStreamingResponse, -} from './codebuddy-chat-stream'; -import { resolveProxyContext } from './codebuddy-context'; +} from './codebuddy/chat-stream'; +import { resolveProxyContext } from './codebuddy/context'; import { buildResponsesBodyFromChat, getUnsupportedResponsesChatOptions, normalizeResponsesUpstreamBody, -} from './codebuddy-responses-request'; +} from './codebuddy/responses-request'; import { mapResponsesPayloadToChat, mapResponsesStreamToChat, -} from './codebuddy-responses-response'; -import { detectServerToolStream } from './codebuddy-server-tools'; +} from './codebuddy/responses-response'; +import { detectServerToolStream } from './codebuddy/server-tools'; import { SERVER_WEB_TOOL_NAMES, type ChatRequestBody, type ProxyContext, -} from './codebuddy-types'; +} from './codebuddy/types'; import { extractResponsesId, extractResponsesUsage, logUpstreamFailure, parseUsageHeader, recordProxyUsage, -} from './codebuddy-usage'; -import { trackResponsesUsageStream } from './codebuddy-usage-stream'; +} from './codebuddy/usage'; +import { trackResponsesUsageStream } from './codebuddy/usage-stream'; import { buildUpstreamBody, buildUpstreamHeaders, headersToRecord, -} from './codebuddy-upstream'; +} from './codebuddy/upstream'; import { enqueueUpstreamResponseSnapshot, setDebugTraceCredential, @@ -671,20 +671,20 @@ export type { ChatRequestBody, DiscoveredModel, ProxyContext, -} from './codebuddy-types'; +} from './codebuddy/types'; export { createProxyContextFromCredential, resolveProxyContext, resolveProxyContextByCredentialFilename, -} from './codebuddy-context'; -export { buildUpstreamHeaders } from './codebuddy-upstream'; +} from './codebuddy/context'; +export { buildUpstreamHeaders } from './codebuddy/upstream'; export { extractImageUrl, isImageContentPart, -} from './codebuddy-responses-request'; +} from './codebuddy/responses-request'; export { getModelsByCredential, getModelsForCredential, getModelsForCredentials, getModelsResponse, -} from './codebuddy-models'; +} from './codebuddy/models'; diff --git a/lib/server/proxy/codebuddy-chat-stream.ts b/lib/server/proxy/codebuddy/chat-stream.ts similarity index 98% rename from lib/server/proxy/codebuddy-chat-stream.ts rename to lib/server/proxy/codebuddy/chat-stream.ts index 5822e53..07c403e 100644 --- a/lib/server/proxy/codebuddy-chat-stream.ts +++ b/lib/server/proxy/codebuddy/chat-stream.ts @@ -1,11 +1,11 @@ -import { createErrorResponse } from '../shared/http'; -import { createSseResponse } from '../shared/sse'; +import { createErrorResponse } from '../../shared/http'; +import { createSseResponse } from '../../shared/sse'; import { chatStreamErrorChunks, createStreamCloser, toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import { recordProxyUsage } from './codebuddy-usage'; +} from '../../shared/upstream-timeout'; +import { recordProxyUsage } from './usage'; import { type ChatStreamChunk, type ChatStreamDelta, @@ -15,7 +15,7 @@ import { type ToolCallChunk, type ToolCallMapping, type ToolCallNormalizationState, -} from './codebuddy-types'; +} from './types'; export const aggregateToolCalls = ( toolCalls: NonNullable, diff --git a/lib/server/proxy/codebuddy-context.ts b/lib/server/proxy/codebuddy/context.ts similarity index 95% rename from lib/server/proxy/codebuddy-context.ts rename to lib/server/proxy/codebuddy/context.ts index 5de5ea3..37e48dc 100644 --- a/lib/server/proxy/codebuddy-context.ts +++ b/lib/server/proxy/codebuddy/context.ts @@ -1,15 +1,15 @@ import type { NextRequest } from 'next/server'; -import { resolveRequestAccessKey } from './auth'; +import { resolveRequestAccessKey } from '../auth'; import { type CredentialRecord, findEligibleCredentialRecordByFilename, findCredentialRecordByFilename, getCredentialProxySettings, resolveCredentialForRequest, -} from '../domain/credentials'; -import { getRequestHeaderMap } from '../shared/http'; -import type { ProxyContext } from './codebuddy-types'; +} from '../../domain/credentials'; +import { getRequestHeaderMap } from '../../shared/http'; +import type { ProxyContext } from './types'; export const getCredentialAffinityKey = ( request: NextRequest, diff --git a/lib/server/proxy/codebuddy-models.ts b/lib/server/proxy/codebuddy/models.ts similarity index 95% rename from lib/server/proxy/codebuddy-models.ts rename to lib/server/proxy/codebuddy/models.ts index 0c9a285..06ca2ef 100644 --- a/lib/server/proxy/codebuddy-models.ts +++ b/lib/server/proxy/codebuddy/models.ts @@ -1,15 +1,15 @@ import type { NextRequest } from 'next/server'; -import { resolveRequestAccessKey } from './auth'; -import { getCodeBuddyApiEndpoint } from '../domain/config'; +import { resolveRequestAccessKey } from '../auth'; +import { getCodeBuddyApiEndpoint } from '../../domain/config'; import { type CredentialData, type CredentialRecord, getCredentialSupportedModels, listEligibleCredentialRecords, -} from '../domain/credentials'; -import { getCredentialValue } from './codebuddy-context'; -import type { DiscoveredModel } from './codebuddy-types'; +} from '../../domain/credentials'; +import { getCredentialValue } from './context'; +import type { DiscoveredModel } from './types'; export const getModelsForCredential = async ({ bearerToken, diff --git a/lib/server/proxy/codebuddy-responses-request.ts b/lib/server/proxy/codebuddy/responses-request.ts similarity index 98% rename from lib/server/proxy/codebuddy-responses-request.ts rename to lib/server/proxy/codebuddy/responses-request.ts index f2444fb..bd6cc96 100644 --- a/lib/server/proxy/codebuddy-responses-request.ts +++ b/lib/server/proxy/codebuddy/responses-request.ts @@ -1,6 +1,6 @@ -import { stringifyContent } from '../shared/content'; -import { resolveHyResponsesReasoning } from '../shared/hy-thought-depth'; -import type { ChatRequestBody } from './codebuddy-types'; +import { stringifyContent } from '../../shared/content'; +import { resolveHyResponsesReasoning } from '../../shared/hy-thought-depth'; +import type { ChatRequestBody } from './types'; export const isImageContentPart = (part: unknown): boolean => { if (!part || typeof part !== 'object') { diff --git a/lib/server/proxy/codebuddy-responses-response.ts b/lib/server/proxy/codebuddy/responses-response.ts similarity index 99% rename from lib/server/proxy/codebuddy-responses-response.ts rename to lib/server/proxy/codebuddy/responses-response.ts index 243552d..76c791d 100644 --- a/lib/server/proxy/codebuddy-responses-response.ts +++ b/lib/server/proxy/codebuddy/responses-response.ts @@ -1,21 +1,21 @@ -import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; import { chatStreamErrorChunks, createStreamCloser, readTimeoutFrame, toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; +} from '../../shared/upstream-timeout'; import { extractResponsesUsage, mapResponsesUsageToChat, parseUsageHeader, recordProxyUsage, -} from './codebuddy-usage'; +} from './usage'; import { CORS_HEADERS, MAX_STREAM_FRAME_LENGTH, type ProxyContext, -} from './codebuddy-types'; +} from './types'; export const normalizeStopSequences = ( stop: string | string[] | undefined, diff --git a/lib/server/proxy/codebuddy-server-tools.ts b/lib/server/proxy/codebuddy/server-tools.ts similarity index 96% rename from lib/server/proxy/codebuddy-server-tools.ts rename to lib/server/proxy/codebuddy/server-tools.ts index b150235..a8e6b66 100644 --- a/lib/server/proxy/codebuddy-server-tools.ts +++ b/lib/server/proxy/codebuddy/server-tools.ts @@ -2,9 +2,9 @@ import { normalizeToolName, WEB_FETCH_TOOL_NAME, WEB_SEARCH_TOOL_NAME, -} from '../search/tool'; -import { aggregateUpstreamStream } from './codebuddy-chat-stream'; -import { type ChatStreamChunk, type StreamProbeState } from './codebuddy-types'; +} from '../../search/tool'; +import { aggregateUpstreamStream } from './chat-stream'; +import { type ChatStreamChunk, type StreamProbeState } from './types'; export const isServerWebToolName = (name: string): boolean => { const normalized = normalizeToolName(name); diff --git a/lib/server/proxy/codebuddy-types.ts b/lib/server/proxy/codebuddy/types.ts similarity index 99% rename from lib/server/proxy/codebuddy-types.ts rename to lib/server/proxy/codebuddy/types.ts index d599996..c79a1a9 100644 --- a/lib/server/proxy/codebuddy-types.ts +++ b/lib/server/proxy/codebuddy/types.ts @@ -2,7 +2,7 @@ import { normalizeToolName, WEB_FETCH_TOOL_NAME, WEB_SEARCH_TOOL_NAME, -} from '../search/tool'; +} from '../../search/tool'; /** * Chat completions are called from browser-side clients as well as servers, so diff --git a/lib/server/proxy/codebuddy-upstream.ts b/lib/server/proxy/codebuddy/upstream.ts similarity index 95% rename from lib/server/proxy/codebuddy-upstream.ts rename to lib/server/proxy/codebuddy/upstream.ts index fe73292..15d7e26 100644 --- a/lib/server/proxy/codebuddy-upstream.ts +++ b/lib/server/proxy/codebuddy/upstream.ts @@ -1,10 +1,10 @@ import type { NextRequest } from 'next/server'; -import { getCodeBuddyApiEndpoint, getDefaultModel } from '../domain/config'; -import { getCredentialSupportedModels } from '../domain/credentials'; -import { resolveHyChatThinking } from '../shared/hy-thought-depth'; -import { getRequestHeaderMap } from '../shared/http'; -import { getCredentialValue } from './codebuddy-context'; +import { getCodeBuddyApiEndpoint, getDefaultModel } from '../../domain/config'; +import { getCredentialSupportedModels } from '../../domain/credentials'; +import { resolveHyChatThinking } from '../../shared/hy-thought-depth'; +import { getRequestHeaderMap } from '../../shared/http'; +import { getCredentialValue } from './context'; import { type CacheableTextBlock, type ChatRequestBody, @@ -14,7 +14,7 @@ import { type OpenAIMessage, type ProxyContext, type ResolvedAuth, -} from './codebuddy-types'; +} from './types'; export const hasPromptCacheControl = (content: unknown): boolean => { return ( diff --git a/lib/server/proxy/codebuddy-usage-stream.ts b/lib/server/proxy/codebuddy/usage-stream.ts similarity index 97% rename from lib/server/proxy/codebuddy-usage-stream.ts rename to lib/server/proxy/codebuddy/usage-stream.ts index cf053ad..05ae238 100644 --- a/lib/server/proxy/codebuddy-usage-stream.ts +++ b/lib/server/proxy/codebuddy/usage-stream.ts @@ -2,13 +2,13 @@ import { createStreamCloser, responsesStreamErrorChunks, toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; +} from '../../shared/upstream-timeout'; import { extractResponsesId, extractResponsesUsage, recordProxyUsage, -} from './codebuddy-usage'; -import { MAX_STREAM_FRAME_LENGTH, type ProxyContext } from './codebuddy-types'; +} from './usage'; +import { MAX_STREAM_FRAME_LENGTH, type ProxyContext } from './types'; export const trackResponsesUsageStream = async ({ fallbackUsage, diff --git a/lib/server/proxy/codebuddy-usage.ts b/lib/server/proxy/codebuddy/usage.ts similarity index 96% rename from lib/server/proxy/codebuddy-usage.ts rename to lib/server/proxy/codebuddy/usage.ts index ece4b71..f814eee 100644 --- a/lib/server/proxy/codebuddy-usage.ts +++ b/lib/server/proxy/codebuddy/usage.ts @@ -1,5 +1,5 @@ -import { recordUsageEvent, type UsageSnapshot } from '../domain/usage'; -import type { ProxyContext } from './codebuddy-types'; +import { recordUsageEvent, type UsageSnapshot } from '../../domain/usage'; +import type { ProxyContext } from './types'; export const toUsageSnapshot = (usage: unknown): UsageSnapshot | null => { if (!usage || typeof usage !== 'object') { diff --git a/lib/server/proxy/responses.ts b/lib/server/proxy/responses.ts index 5a8ec2c..0bd4641 100644 --- a/lib/server/proxy/responses.ts +++ b/lib/server/proxy/responses.ts @@ -25,22 +25,22 @@ import { resolveProxyContextByCredentialFilename, } from './codebuddy'; import { executeImageGenerationLoop } from './image-generation'; -import { mapChatResponseToResponsesPayload } from './responses-payload'; +import { mapChatResponseToResponsesPayload } from './responses/payload'; import { getResponseSession, getValidatedPreviousSession, storeUpstreamResponseBinding, -} from './responses-session'; -import { createResponsesEventStream } from './responses-event-stream'; +} from './responses/session'; +import { createResponsesEventStream } from './responses/event-stream'; import { getResponsesCompatibilityError, hasImageGenerationTool, normalizeTranscriptMessageToolNames, translateResponsesToolsToChat, translateResponsesToolChoiceToChatWithTools, -} from './responses-tools'; -import { prepareTranscript } from './responses-transcript'; -import type { ResponsesRequestBody } from './responses-types'; +} from './responses/tools'; +import { prepareTranscript } from './responses/transcript'; +import type { ResponsesRequestBody } from './responses/types'; import { getServerToolExecutions } from './web-search-loop'; export const handleResponsesRequest = async ( @@ -283,5 +283,5 @@ export const handleResponsesRequest = async ( } }; -export { translateResponsesToolsToChat } from './responses-tools'; -export { resetResponseSessions } from './responses-session'; +export { translateResponsesToolsToChat } from './responses/tools'; +export { resetResponseSessions } from './responses/session'; diff --git a/lib/server/proxy/responses-event-stream.ts b/lib/server/proxy/responses/event-stream.ts similarity index 94% rename from lib/server/proxy/responses-event-stream.ts rename to lib/server/proxy/responses/event-stream.ts index 40fe131..0e3a3ba 100644 --- a/lib/server/proxy/responses-event-stream.ts +++ b/lib/server/proxy/responses/event-stream.ts @@ -8,34 +8,34 @@ import type { NextRequest } from 'next/server'; -import { isWebFetchEnabled, isWebSearchEnabled } from '../domain/config'; -import type { DebugTrace } from '../domain/debug'; +import { isWebFetchEnabled, isWebSearchEnabled } from '../../domain/config'; +import type { DebugTrace } from '../../domain/debug'; import { normalizeToolName, WEB_FETCH_TOOL_NAME, WEB_SEARCH_TOOL_NAME, -} from '../search/tool'; -import { createSseResponse, encodeDoneFrame } from '../shared/sse'; -import { proxyChatCompletions, type ProxyContext } from './codebuddy'; -import { executeImageGenerationLoop } from './image-generation'; +} from '../../search/tool'; +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; +import { proxyChatCompletions, type ProxyContext } from '../codebuddy'; +import { executeImageGenerationLoop } from '../image-generation'; import { buildResponsesWebSearchCallItem, mapChatResponseToResponsesStream, -} from './responses-payload'; -import { createResponseId } from './responses-ids'; +} from './payload'; +import { createResponseId } from './ids'; -import { mapChatStreamToResponsesEventStream } from './responses-stream'; +import { mapChatStreamToResponsesEventStream } from './stream'; import { hasImageGenerationTool, normalizeTranscriptMessageToolNames, translateResponsesToolsToChat, translateResponsesToolChoiceToChatWithTools, -} from './responses-tools'; +} from './tools'; import type { ResponsesServerToolItem, ResponseSessionDefaults, TranscriptMessage, -} from './responses-types'; +} from './types'; export const createResponsesEventStream = async ( request: NextRequest, diff --git a/lib/server/proxy/responses-ids.ts b/lib/server/proxy/responses/ids.ts similarity index 100% rename from lib/server/proxy/responses-ids.ts rename to lib/server/proxy/responses/ids.ts diff --git a/lib/server/proxy/responses-payload.ts b/lib/server/proxy/responses/payload.ts similarity index 96% rename from lib/server/proxy/responses-payload.ts rename to lib/server/proxy/responses/payload.ts index e0bf4be..c738d54 100644 --- a/lib/server/proxy/responses-payload.ts +++ b/lib/server/proxy/responses/payload.ts @@ -2,41 +2,41 @@ // Non-streaming Responses payload assembly // --------------------------------------------------------------------------- -import { stringifyContent } from '../shared/content'; +import { stringifyContent } from '../../shared/content'; import { createSseResponse, DONE_FRAME_TEXT, eventFrameText, -} from '../shared/sse'; -import type { ProxyContext } from './codebuddy'; +} from '../../shared/sse'; +import type { ProxyContext } from '../codebuddy'; import { buildResponsesImageGenerationCallItem, type ImageGenerationExecution, -} from './image-generation'; +} from '../image-generation'; import { createMessageId, createResponseId, createResponseOutputId, createResponseReasoningId, normalizeToolCallId, -} from './responses-ids'; -import { storeResponseSession } from './responses-session'; -import { buildResponsesToolCallOutputItem } from './responses-tools'; +} from './ids'; +import { storeResponseSession } from './session'; +import { buildResponsesToolCallOutputItem } from './tools'; import { buildAssistantTranscriptToolCalls, getAssistantTranscriptContent, mapChatUsageToResponses, REASONING_PREFIX, -} from './responses-transcript'; +} from './transcript'; import type { ChatResponseMessage, ResponseSessionDefaults, TranscriptMessage, -} from './responses-types'; +} from './types'; import type { ServerToolExecution, ServerToolInvocation, -} from './web-search-loop'; +} from '../web-search-loop'; export const mapChatResponseToResponsesPayload = async ( accessKeyId: string | null, diff --git a/lib/server/proxy/responses-session.ts b/lib/server/proxy/responses/session.ts similarity index 97% rename from lib/server/proxy/responses-session.ts rename to lib/server/proxy/responses/session.ts index cefc1ef..fd0d25e 100644 --- a/lib/server/proxy/responses-session.ts +++ b/lib/server/proxy/responses/session.ts @@ -2,18 +2,15 @@ // Response session persistence // --------------------------------------------------------------------------- -import type { ProxyContext } from './codebuddy'; +import type { ProxyContext } from '../codebuddy'; import { deleteStorageJson, getStorageBackendMeta, listStorageJson, readStorageJson, writeStorageJson, -} from '../storage'; -import type { - ResponseSession, - ResponseSessionMetadata, -} from './responses-types'; +} from '../../storage'; +import type { ResponseSession, ResponseSessionMetadata } from './types'; export const MAX_RESPONSE_SESSIONS = 1_000; export const RESPONSE_SESSION_TTL_MS = 60 * 60 * 1000; diff --git a/lib/server/proxy/responses-stream.ts b/lib/server/proxy/responses/stream.ts similarity index 97% rename from lib/server/proxy/responses-stream.ts rename to lib/server/proxy/responses/stream.ts index 0c21f39..1e490db 100644 --- a/lib/server/proxy/responses-stream.ts +++ b/lib/server/proxy/responses/stream.ts @@ -2,29 +2,29 @@ // Responses SSE stream mapping // --------------------------------------------------------------------------- -import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; import { createStreamCloser, readTimeoutFrame, responsesStreamErrorChunks, toUpstreamTimeoutMessage, -} from '../shared/upstream-timeout'; -import type { ProxyContext } from './codebuddy'; +} from '../../shared/upstream-timeout'; +import type { ProxyContext } from '../codebuddy'; -import { storeResponseSession } from './responses-session'; -import { buildResponsesWebSearchCallItem } from './responses-payload'; +import { storeResponseSession } from './session'; +import { buildResponsesWebSearchCallItem } from './payload'; import { createMessageId, createResponseId, createResponseOutputId, createResponseReasoningId, normalizeToolCallId, -} from './responses-ids'; +} from './ids'; import { buildResponsesToolCallOutputItem, getResponsesToolCallArgumentDeltaEventType, hasSupportedLongerToolNamePrefix, -} from './responses-tools'; +} from './tools'; import { buildStreamingAssistantTranscriptToolCalls, getAssistantTranscriptContent, @@ -32,7 +32,7 @@ import { getStreamingToolCallLookupKeys, mapChatUsageToResponses, REASONING_PREFIX, -} from './responses-transcript'; +} from './transcript'; import type { ChatResponseToolCall, ResponsesServerToolItem, @@ -40,9 +40,9 @@ import type { StreamingMessageState, StreamingToolCallState, TranscriptMessage, -} from './responses-types'; -import { getServerToolExecutions } from './web-search-loop'; -import { MAX_RESPONSE_SESSION_TOTAL_BYTES } from './responses-session'; +} from './types'; +import { getServerToolExecutions } from '../web-search-loop'; +import { MAX_RESPONSE_SESSION_TOTAL_BYTES } from './session'; const MAX_STREAM_BUFFER_LENGTH = 1_000_000; const MAX_STREAM_TEXT_LENGTH = 2_000_000; diff --git a/lib/server/proxy/responses-tools.ts b/lib/server/proxy/responses/tools.ts similarity index 99% rename from lib/server/proxy/responses-tools.ts rename to lib/server/proxy/responses/tools.ts index 903a1a7..0124d22 100644 --- a/lib/server/proxy/responses-tools.ts +++ b/lib/server/proxy/responses/tools.ts @@ -2,7 +2,7 @@ // Responses <-> Chat tool translation // --------------------------------------------------------------------------- -import { createErrorResponse } from '../shared/http'; +import { createErrorResponse } from '../../shared/http'; import { buildWebFetchToolDefinition, buildWebSearchToolDefinition, @@ -12,18 +12,18 @@ import { WEB_FETCH_TOOL_TYPE_PREFIX, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; +} from '../../search/tool'; import { buildImageGenerationChatTool, IMAGE_GENERATION_CHAT_TOOL_NAME, IMAGE_GENERATION_TOOL_TYPE, -} from './image-generation'; +} from '../image-generation'; import type { ResponsesRequestBody, SupportedChatTool, SupportedResponsesTool, TranscriptMessage, -} from './responses-types'; +} from './types'; export const TOOL_SEARCH_PROXY_NAME = 'tool_search'; export const CUSTOM_TOOL_INPUT_FIELD = 'input'; diff --git a/lib/server/proxy/responses-transcript.ts b/lib/server/proxy/responses/transcript.ts similarity index 97% rename from lib/server/proxy/responses-transcript.ts rename to lib/server/proxy/responses/transcript.ts index 9af7c2c..9744f6c 100644 --- a/lib/server/proxy/responses-transcript.ts +++ b/lib/server/proxy/responses/transcript.ts @@ -2,15 +2,15 @@ // Transcript construction and usage mapping // --------------------------------------------------------------------------- -import { getDefaultModel } from '../domain/config'; -import { stringifyContent } from '../shared/content'; -import { extractImageUrl, isImageContentPart } from './codebuddy'; -import { createResponseOutputId, normalizeToolCallId } from './responses-ids'; +import { getDefaultModel } from '../../domain/config'; +import { stringifyContent } from '../../shared/content'; +import { extractImageUrl, isImageContentPart } from '../codebuddy'; +import { createResponseOutputId, normalizeToolCallId } from './ids'; import { getValidatedPreviousSession, MAX_RESPONSE_TRANSCRIPT_MESSAGES, -} from './responses-session'; -import { findSupportedToolByName } from './responses-tools'; +} from './session'; +import { findSupportedToolByName } from './tools'; import type { ChatContentPart, ChatResponseToolCall, @@ -21,7 +21,7 @@ import type { StreamingToolCallState, TranscriptContent, TranscriptMessage, -} from './responses-types'; +} from './types'; export const buildAssistantTranscriptToolCalls = ( toolCalls: ChatResponseToolCall[], diff --git a/lib/server/proxy/responses-types.ts b/lib/server/proxy/responses/types.ts similarity index 100% rename from lib/server/proxy/responses-types.ts rename to lib/server/proxy/responses/types.ts diff --git a/lib/server/proxy/server-tool-args.ts b/lib/server/proxy/server-tool/args.ts similarity index 96% rename from lib/server/proxy/server-tool-args.ts rename to lib/server/proxy/server-tool/args.ts index 5741ebd..209bdf8 100644 --- a/lib/server/proxy/server-tool-args.ts +++ b/lib/server/proxy/server-tool/args.ts @@ -1,5 +1,5 @@ -import { asRecord } from '../shared/content'; -import type { WebFetchQuery } from '../search/types'; +import { asRecord } from '../../shared/content'; +import type { WebFetchQuery } from '../../search/types'; /** * Reads one string field out of a tool-call argument object. diff --git a/lib/server/proxy/server-tool-classify.ts b/lib/server/proxy/server-tool/classify.ts similarity index 97% rename from lib/server/proxy/server-tool-classify.ts rename to lib/server/proxy/server-tool/classify.ts index 45daa2d..8fce0ac 100644 --- a/lib/server/proxy/server-tool-classify.ts +++ b/lib/server/proxy/server-tool/classify.ts @@ -1,4 +1,4 @@ -import { asRecord } from '../shared/content'; +import { asRecord } from '../../shared/content'; import { buildWebFetchToolDefinition, buildWebSearchToolDefinition, @@ -9,9 +9,9 @@ import { WEB_FETCH_TOOL_TYPE_PREFIX, WEB_SEARCH_TOOL_NAME, WEB_SEARCH_TOOL_TYPE_PREFIX, -} from '../search/tool'; -import type { WebFetchProvider, WebSearchProvider } from '../search/types'; -import type { ChatCompletionToolCall } from './server-tool-types'; +} from '../../search/tool'; +import type { WebFetchProvider, WebSearchProvider } from '../../search/types'; +import type { ChatCompletionToolCall } from './types'; /** * Recognises one server-tool declaration. diff --git a/lib/server/proxy/server-tool-execution.ts b/lib/server/proxy/server-tool/execution.ts similarity index 91% rename from lib/server/proxy/server-tool-execution.ts rename to lib/server/proxy/server-tool/execution.ts index af047e1..839d4c8 100644 --- a/lib/server/proxy/server-tool-execution.ts +++ b/lib/server/proxy/server-tool/execution.ts @@ -1,8 +1,8 @@ -import { runWebFetchResult, runWebSearchResult } from '../search'; -import type { WebFetchProvider, WebSearchProvider } from '../search/types'; -import { asRecord } from '../shared/content'; -import { extractFetchQuery, extractSearchQuery } from './server-tool-args'; -import { isWebFetchToolCall } from './server-tool-classify'; +import { runWebFetchResult, runWebSearchResult } from '../../search'; +import type { WebFetchProvider, WebSearchProvider } from '../../search/types'; +import { asRecord } from '../../shared/content'; +import { extractFetchQuery, extractSearchQuery } from './args'; +import { isWebFetchToolCall } from './classify'; import { SERVER_TOOL_STREAM_EVENT_KEY, type ChatCompletionToolCall, @@ -11,7 +11,7 @@ import { type ServerToolInvocation, type ServerToolStreamEvent, type ServerToolTurn, -} from './server-tool-types'; +} from './types'; export const getServerToolStreamEvent = ( value: unknown, diff --git a/lib/server/proxy/server-tool-payload.ts b/lib/server/proxy/server-tool/payload.ts similarity index 96% rename from lib/server/proxy/server-tool-payload.ts rename to lib/server/proxy/server-tool/payload.ts index 0538e01..1339645 100644 --- a/lib/server/proxy/server-tool-payload.ts +++ b/lib/server/proxy/server-tool/payload.ts @@ -1,5 +1,5 @@ -import { extractErrorMessage } from '../shared/http'; -import type { ChatCompletionPayload } from './server-tool-types'; +import { extractErrorMessage } from '../../shared/http'; +import type { ChatCompletionPayload } from './types'; /** * Rebuilds a failed upstream response so its body can be read again. diff --git a/lib/server/proxy/server-tool-sse.ts b/lib/server/proxy/server-tool/sse.ts similarity index 97% rename from lib/server/proxy/server-tool-sse.ts rename to lib/server/proxy/server-tool/sse.ts index 413c4bc..7a87c86 100644 --- a/lib/server/proxy/server-tool-sse.ts +++ b/lib/server/proxy/server-tool/sse.ts @@ -1,9 +1,9 @@ -import { createSseResponse, encodeDoneFrame } from '../shared/sse'; +import { createSseResponse, encodeDoneFrame } from '../../shared/sse'; import { STREAM_TEXT_CHUNK_LENGTH, type ChatCompletionPayload, type JsonRecord, -} from './server-tool-types'; +} from './types'; /** * Replays a buffered completion as chat-completion SSE. Used only after a diff --git a/lib/server/proxy/server-tool-stream.ts b/lib/server/proxy/server-tool/stream.ts similarity index 96% rename from lib/server/proxy/server-tool-stream.ts rename to lib/server/proxy/server-tool/stream.ts index a9c1bda..8233816 100644 --- a/lib/server/proxy/server-tool-stream.ts +++ b/lib/server/proxy/server-tool/stream.ts @@ -1,9 +1,6 @@ -import { isLocalServerToolCall } from './server-tool-classify'; -import type { - ChatCompletionMessage, - ChatCompletionToolCall, -} from './server-tool-types'; -import type { WebFetchProvider, WebSearchProvider } from '../search/types'; +import { isLocalServerToolCall } from './classify'; +import type { ChatCompletionMessage, ChatCompletionToolCall } from './types'; +import type { WebFetchProvider, WebSearchProvider } from '../../search/types'; export const mergeStreamingToolName = ( previous: string, diff --git a/lib/server/proxy/server-tool-turns.ts b/lib/server/proxy/server-tool/turns.ts similarity index 98% rename from lib/server/proxy/server-tool-turns.ts rename to lib/server/proxy/server-tool/turns.ts index 3d31a19..877b9e2 100644 --- a/lib/server/proxy/server-tool-turns.ts +++ b/lib/server/proxy/server-tool/turns.ts @@ -1,4 +1,4 @@ -import { asRecord, readReasoning } from '../shared/content'; +import { asRecord, readReasoning } from '../../shared/content'; import type { ChatCompletionMessage, ChatCompletionPayload, @@ -6,7 +6,7 @@ import type { JsonRecord, ServerToolExecution, ServerToolTurn, -} from './server-tool-types'; +} from './types'; export const sumUsage = (accumulated: unknown, incoming: unknown): unknown => { const left = asRecord(accumulated); diff --git a/lib/server/proxy/server-tool-types.ts b/lib/server/proxy/server-tool/types.ts similarity index 97% rename from lib/server/proxy/server-tool-types.ts rename to lib/server/proxy/server-tool/types.ts index 3ca1e50..89c5c97 100644 --- a/lib/server/proxy/server-tool-types.ts +++ b/lib/server/proxy/server-tool/types.ts @@ -2,8 +2,8 @@ import type { WebFetchQuery, WebFetchResponse, WebSearchResponse, -} from '../search/types'; -import type { ChatRequestBody } from './codebuddy'; +} from '../../search/types'; +import type { ChatRequestBody } from '../codebuddy'; export const MAX_SEARCH_ITERATIONS = 5; export const STREAM_TEXT_CHUNK_LENGTH = 1024; diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index 93a4360..92ee8a6 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -15,27 +15,27 @@ import { isWebFetchTool, isWebSearchTool, replaceServerTools, -} from './server-tool-classify'; +} from './server-tool/classify'; import { buildServerToolInvocation, executeServerToolInvocations, -} from './server-tool-execution'; +} from './server-tool/execution'; import { buildServerToolFailureResponse, parseBufferedPayload, readBufferedChatCompletionPayload, -} from './server-tool-payload'; -import { synthesizeChatCompletionStream } from './server-tool-sse'; +} from './server-tool/payload'; +import { synthesizeChatCompletionStream } from './server-tool/sse'; import { type ServerToolProbe, probeServerToolStream, -} from './server-tool-stream'; +} from './server-tool/stream'; import { buildMixedTurnPayload, readReasoning, sumUsage, withIntermediateTurns, -} from './server-tool-turns'; +} from './server-tool/turns'; import { MAX_SEARCH_ITERATIONS, SERVER_TOOL_STREAM_EVENT_KEY, @@ -49,7 +49,7 @@ import { type ServerToolStreamEvent, type ServerToolTurn, type ServerToolUpstreamMode, -} from './server-tool-types'; +} from './server-tool/types'; /** * Server-side web search for upstreams that do not implement it. @@ -909,7 +909,7 @@ export type { ServerToolStreamEvent, ServerToolTurn, ServerToolUpstreamMode, -} from './server-tool-types'; +} from './server-tool/types'; export { attachServerToolExecutions, @@ -917,8 +917,8 @@ export { getServerToolExecutions, getServerToolStreamEvent, getServerToolTurns, -} from './server-tool-execution'; +} from './server-tool/execution'; -export { synthesizeChatCompletionStream } from './server-tool-sse'; +export { synthesizeChatCompletionStream } from './server-tool/sse'; -export { withIntermediateTurns } from './server-tool-turns'; +export { withIntermediateTurns } from './server-tool/turns';