diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index b88c9de..cb83eb3 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -1283,9 +1283,23 @@ const createAnthropicServerToolEventStream = ( } if (!upstreamResponse.ok || !upstreamResponse.body) { + // A rate limit has to arrive as `rate_limit_error`, or a client that + // retries on that type alone will treat an exhausted quota as a + // generic failure and stop retrying — so the upstream status drives + // the event type even though the envelope is already streaming and + // the HTTP status cannot be changed. + const message = upstreamResponse.ok + ? 'Upstream request failed' + : await getUpstreamErrorMessage(upstreamResponse).catch( + () => 'Upstream request failed', + ); + enqueueEvent({ type: 'error', - error: { type: 'api_error', message: 'Upstream request failed' }, + error: { + type: anthropicErrorType(upstreamResponse.status), + message, + }, }); controller.close(); return; @@ -1429,26 +1443,28 @@ export const handleMessagesRequest = async ( } }; +export const anthropicErrorType = (status: number): string => + status === 401 + ? 'authentication_error' + : status === 403 + ? 'permission_error' + : status === 404 + ? 'not_found_error' + : status === 413 + ? 'request_too_large' + : status === 429 + ? 'rate_limit_error' + : status === 529 + ? 'overloaded_error' + : status >= 500 + ? 'api_error' + : 'invalid_request_error'; + export const createAnthropicError = ( status: number, message: string, ): Response => { - const type = - status === 401 - ? 'authentication_error' - : status === 403 - ? 'permission_error' - : status === 404 - ? 'not_found_error' - : status === 413 - ? 'request_too_large' - : status === 429 - ? 'rate_limit_error' - : status === 529 - ? 'overloaded_error' - : status >= 500 - ? 'api_error' - : 'invalid_request_error'; + const type = anthropicErrorType(status); return Response.json( { diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index 60bbf5b..74ac613 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -85,6 +85,38 @@ export interface ChatCompletionPayload { usage?: unknown; } +/** + * Rebuilds a failed upstream response so its body can be read again. + * + * A `Response` body can only be consumed once. The loop reads it to decide + * whether the model asked for a server tool, and handing the same object back + * used to leave the route layer — which reads it again to build the answer the + * client actually sees — with a spent body: the second read threw + * "Body already used" and the client got a 500 in place of the real upstream + * status. Draining it here and replaying the bytes in a fresh response keeps + * both reads working and preserves the body verbatim, so an upstream error + * detail that is not valid JSON still reaches the client intact. + * + * `content-length` and `content-encoding` are dropped: the body is re-emitted + * rather than re-encoded, and a stale length would describe bytes the upstream + * compressed before this layer ever saw them. + */ +const buildServerToolFailureResponse = async ( + response: Response, +): Promise => { + const headers = new Headers(response.headers); + + headers.delete('content-length'); + headers.delete('content-encoding'); + headers.set('content-type', 'application/json'); + + return new Response(await response.text(), { + headers, + status: response.status, + statusText: response.statusText, + }); +}; + const readBufferedChatCompletionPayload = async ( response: Response, ): Promise => { @@ -462,6 +494,68 @@ const buildMixedTurnPayload = ({ }; }; +const readReasoning = (message: ChatCompletionMessage | undefined): string => { + if (!message) { + return ''; + } + + if (typeof message.reasoning_content === 'string') { + return message.reasoning_content; + } + + return typeof message.reasoning === 'string' ? message.reasoning : ''; +}; + +/** + * Folds the text a multi-hop turn produced before its later server-tool calls + * into the payload the client receives. + * + * Only the last iteration's message is in `payload`, but a turn that searched + * more than once spoke before each search, and that text is part of the turn: + * dropping it hides the model's reasoning from the user and leaves the + * client's transcript out of step with what the model actually said. + */ +const withIntermediateTurns = ({ + payload, + reasonings, + texts, +}: { + payload: ChatCompletionPayload; + reasonings: string[]; + texts: string[]; +}): ChatCompletionPayload => { + const extraText = texts.filter(Boolean).join('\n\n'); + const extraReasoning = reasonings.filter(Boolean).join('\n\n'); + const [first, ...rest] = payload.choices ?? []; + + if (!first || (!extraText && !extraReasoning)) { + return payload; + } + + const message = first.message ?? {}; + const existingText = + typeof message.content === 'string' ? message.content : ''; + const content = [extraText, existingText].filter(Boolean).join('\n\n'); + const reasoning = [extraReasoning, readReasoning(message)] + .filter(Boolean) + .join('\n\n'); + + return { + ...payload, + choices: [ + { + ...first, + message: { + ...message, + content, + ...(reasoning ? { reasoning_content: reasoning } : {}), + }, + }, + ...rest, + ], + }; +}; + /** * Result of one server-tool pass. * @@ -670,6 +764,191 @@ const executeServerToolInvocations = async ({ ); }; +/** + * Result of streaming one upstream response while watching for server-tool + * calls, so a follow-up iteration can decide what happened. + * + * `localCalls` and `remainingCalls` partition the aggregated tool calls the + * way the execution loop needs them; `frames` are the frames that were held + * back because they carried tool-call deltas. + */ +interface ServerToolProbe { + content: string; + frames: string[]; + localCalls: ChatCompletionToolCall[]; + reasoning: string; + remainingCalls: ChatCompletionToolCall[]; + role: string; + toolCalls: ChatCompletionToolCall[]; + usage: unknown; +} + +/** + * Whether the proxy is the one meant to answer this call. + * + * A call without an available backend is not a fallback to the client — it + * leaves the loop as an unanswered client-owned tool — but it must not be + * counted as locally executable either. + */ +const isLocalServerToolCall = ({ + fetchProvider, + toolCall, + searchProvider, +}: { + fetchProvider: WebFetchProvider | null; + toolCall: ChatCompletionToolCall; + searchProvider: WebSearchProvider | null; +}): boolean => + (Boolean(searchProvider) && isWebSearchToolCall(toolCall)) || + (Boolean(fetchProvider) && isWebFetchToolCall(toolCall)); + +const probeServerToolStream = async ({ + canContinue, + context, + emitRaw, + fetchProvider, + onReader, + response, + searchProvider, +}: { + canContinue: () => boolean; + context: { + responseCreated: number; + responseId: string; + responseModel: string; + responseObject: string; + role: string; + usage: unknown; + }; + emitRaw: (frame: string) => void; + fetchProvider: WebFetchProvider | null; + /** + * Hands the active reader to the caller's cancellation path. Without it a + * disconnect cannot interrupt a read that is already parked: the loop only + * notices the cancellation once upstream produces another chunk, which a + * stalled upstream never does. + */ + onReader?: (reader: ReadableStreamDefaultReader | null) => void; + response: Response; + searchProvider: WebSearchProvider | null; +}): Promise => { + const frames: string[] = []; + const toolCallDeltas: ChatCompletionToolCall[] = []; + const decoder = new TextDecoder(); + const reader = response.body!.getReader(); + onReader?.(reader); + let buffer = ''; + let content = ''; + let reasoning = ''; + + const inspectFrame = (frame: string): void => { + const line = frame + .split(/\r?\n/) + .find((segment) => segment.startsWith('data:')); + + if (!line) { + emitRaw(frame); + return; + } + + const raw = line.slice(5).trim(); + if (!raw) return; + if (raw === '[DONE]') { + frames.push(frame); + return; + } + + try { + const chunk = JSON.parse(raw) as { + choices?: Array<{ + delta?: ChatCompletionMessage & { + tool_calls?: ChatCompletionToolCall[]; + }; + finish_reason?: string | null; + }>; + created?: number; + id?: string; + model?: string; + object?: string; + usage?: unknown; + }; + context.responseId = chunk.id ?? context.responseId; + context.responseModel = chunk.model ?? context.responseModel; + context.responseObject = + chunk.object?.replace(/\.chunk$/, '') ?? context.responseObject; + context.responseCreated = chunk.created ?? context.responseCreated; + context.usage = chunk.usage ?? context.usage; + const choice = chunk.choices?.[0]; + const delta = choice?.delta; + context.role = delta?.role ?? context.role; + content += delta?.content ?? ''; + reasoning += delta?.reasoning_content ?? delta?.reasoning ?? ''; + + // A tool-call frame is held rather than forwarded: if the turn turns out + // to invoke a server tool, the call has to be answered locally instead + // of being handed to the client as an unresolved call. Anything else the + // delta carried — most importantly the text the model wrote before + // deciding to search — still belongs to the visible turn, so it is + // re-emitted without the tool call. + if (delta?.tool_calls?.length) { + toolCallDeltas.push(...delta.tool_calls); + frames.push(frame); + + const visibleDelta = { ...delta }; + delete visibleDelta.tool_calls; + + if (Object.keys(visibleDelta).length) { + const visible = JSON.stringify({ + ...chunk, + choices: [{ ...choice, delta: visibleDelta, finish_reason: null }], + }); + emitRaw(`data: ${visible}`); + } + return; + } + + if (choice?.finish_reason === 'tool_calls') { + frames.push(frame); + return; + } + } catch { + emitRaw(frame); + return; + } + + emitRaw(frame); + }; + + while (true) { + const chunk = await reader.read(); + if (!canContinue()) break; + if (chunk.done) break; + buffer += decoder.decode(chunk.value, { stream: true }); + const split = buffer.split(/\r?\n\r?\n/); + buffer = split.pop() ?? ''; + split.forEach(inspectFrame); + } + + if (buffer.trim()) inspectFrame(buffer); + reader.releaseLock(); + onReader?.(null); + + const toolCalls = aggregateStreamingToolCalls(toolCallDeltas); + const isLocalCall = (toolCall: ChatCompletionToolCall): boolean => + isLocalServerToolCall({ fetchProvider, searchProvider, toolCall }); + + return { + content, + frames, + localCalls: toolCalls.filter(isLocalCall), + reasoning, + remainingCalls: toolCalls.filter((toolCall) => !isLocalCall(toolCall)), + role: context.role, + toolCalls, + usage: context.usage, + }; +}; + const createInlineServerToolStream = async ({ body, callbacks, @@ -697,6 +976,12 @@ const createInlineServerToolStream = async ({ const encoder = new TextEncoder(); let activeReader: ReadableStreamDefaultReader | null = null; let cancelled = false; + + // A call is locally executable only when its backend is available; anything + // else stays the client's to answer. + const isLocalCall = (toolCall: ChatCompletionToolCall): boolean => + isLocalServerToolCall({ fetchProvider, searchProvider, toolCall }); + const emitJson = ( controller: ReadableStreamDefaultController, payload: Record, @@ -730,129 +1015,52 @@ const createInlineServerToolStream = async ({ const stream = new ReadableStream({ start: (controller) => { const run = async (): Promise => { - const reader = firstResponse.body!.getReader(); - activeReader = reader; - const decoder = new TextDecoder(); - const heldToolFrames: string[] = []; - const toolCallDeltas: ChatCompletionToolCall[] = []; - let buffer = ''; - let responseId = ''; - let responseModel = String(body.model ?? 'unknown'); - let responseObject = 'chat.completion'; - let responseCreated = Math.floor(Date.now() / 1000); - let role = 'assistant'; - let content = ''; - let reasoning = ''; - let usage: unknown = null; - - const inspectFrame = (frame: string): void => { - const line = frame - .split(/\r?\n/) - .find((segment) => segment.startsWith('data:')); - - if (!line) { - controller.enqueue(encoder.encode(`${frame}\n\n`)); - return; - } - - const raw = line.slice(5).trim(); - if (!raw) return; - if (raw === '[DONE]') { - heldToolFrames.push(frame); - return; - } - - try { - const chunk = JSON.parse(raw) as { - choices?: Array<{ - delta?: ChatCompletionMessage & { - tool_calls?: ChatCompletionToolCall[]; - }; - finish_reason?: string | null; - }>; - created?: number; - id?: string; - model?: string; - object?: string; - usage?: unknown; - }; - responseId = chunk.id ?? responseId; - responseModel = chunk.model ?? responseModel; - responseObject = - chunk.object?.replace(/\.chunk$/, '') ?? responseObject; - responseCreated = chunk.created ?? responseCreated; - usage = chunk.usage ?? usage; - const choice = chunk.choices?.[0]; - const delta = choice?.delta; - role = delta?.role ?? role; - content += delta?.content ?? ''; - reasoning += delta?.reasoning_content ?? delta?.reasoning ?? ''; - - if (delta?.tool_calls?.length) { - toolCallDeltas.push(...delta.tool_calls); - heldToolFrames.push(frame); - - const visibleDelta = { ...delta }; - delete visibleDelta.tool_calls; - - if (Object.keys(visibleDelta).length) { - emitJson(controller, { - ...chunk, - choices: [ - { ...choice, delta: visibleDelta, finish_reason: null }, - ], - }); - } - return; - } - - if (choice?.finish_reason === 'tool_calls') { - heldToolFrames.push(frame); - return; - } - } catch { - controller.enqueue(encoder.encode(`${frame}\n\n`)); - return; - } - - controller.enqueue(encoder.encode(`${frame}\n\n`)); + activeReader = firstResponse.body!.getReader(); + activeReader.releaseLock(); + const context = { + responseCreated: Math.floor(Date.now() / 1000), + responseId: '', + responseModel: String(body.model ?? 'unknown'), + responseObject: 'chat.completion', + role: 'assistant', + usage: null as unknown, }; - while (true) { - const chunk = await reader.read(); - if (cancelled) return; - if (chunk.done) break; - buffer += decoder.decode(chunk.value, { stream: true }); - const frames = buffer.split(/\r?\n\r?\n/); - buffer = frames.pop() ?? ''; - frames.forEach(inspectFrame); - } - - if (buffer.trim()) inspectFrame(buffer); - reader.releaseLock(); + const first = await probeServerToolStream({ + canContinue: () => !cancelled, + context, + emitRaw: (frame) => + controller.enqueue(encoder.encode(`${frame}\n\n`)), + fetchProvider, + onReader: (reader) => { + activeReader = reader; + }, + response: firstResponse, + searchProvider, + }); + if (cancelled) return; activeReader = null; - const toolCalls = aggregateStreamingToolCalls(toolCallDeltas); - const localCalls = toolCalls.filter( - (toolCall) => - (Boolean(searchProvider) && isWebSearchToolCall(toolCall)) || - (Boolean(fetchProvider) && isWebFetchToolCall(toolCall)), - ); + let usage: unknown = context.usage; + const content = first.content; + const reasoning = first.reasoning; + const role = context.role; + + const responseId = context.responseId; + const responseModel = context.responseModel; + const responseObject = context.responseObject; + const responseCreated = context.responseCreated; - if (!localCalls.length) { - heldToolFrames.forEach((frame) => + if (!first.localCalls.length) { + first.frames.forEach((frame) => controller.enqueue(encoder.encode(`${frame}\n\n`)), ); controller.close(); return; } - const remainingCalls = toolCalls.filter( - (toolCall) => - (!searchProvider || !isWebSearchToolCall(toolCall)) && - (!fetchProvider || !isWebFetchToolCall(toolCall)), - ); - const invocations = localCalls.map((toolCall, index) => + const remainingCalls = first.remainingCalls; + const invocations = first.localCalls.map((toolCall, index) => buildServerToolInvocation(toolCall, 0, index), ); @@ -908,7 +1116,7 @@ const createInlineServerToolStream = async ({ const assistantMessage: JsonRecord = { role, content: content || null, - tool_calls: toolCalls, + tool_calls: first.toolCalls, ...(reasoning ? { reasoning_content: reasoning } : {}), }; messages.push(assistantMessage); @@ -932,35 +1140,131 @@ const createInlineServerToolStream = async ({ iteration < MAX_SEARCH_ITERATIONS; iteration++ ) { - const response = await callUpstream(loopBody, 'buffer'); - const payload = await readBufferedChatCompletionPayload(response); - - if (!response.ok || payload.error) { - emitJson(controller, payload as JsonRecord); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - controller.close(); - return; + const response = await callUpstream(loopBody, 'stream'); + const isEventStream = (response.headers.get('content-type') ?? '') + .toLowerCase() + .includes('text/event-stream'); + + // Upstream answers with JSON rather than SSE when it refuses the + // request, and also when the caller is not streaming at all. Both + // shapes are read the same way; only an error ends the turn here. + const buffered = !isEventStream + ? await readBufferedChatCompletionPayload(response) + : null; + + let probe: ServerToolProbe | null = null; + + if (buffered) { + usage = sumUsage(usage, buffered.usage); + + if (!response.ok || buffered.error) { + emitJson(controller, buffered as JsonRecord); + controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.close(); + return; + } + + const bufferedMessage = buffered.choices?.[0]?.message; + const bufferedCalls = bufferedMessage?.tool_calls ?? []; + + // A JSON answer that still asks for a server tool is an + // intermediate step, not the end of the turn: it has to be + // executed and fed back, exactly as a streamed one would be. + if (!bufferedCalls.some(isLocalCall)) { + finalPayload = { + ...buffered, + ...(usage ? { usage } : {}), + }; + break; + } + + probe = { + content: + typeof bufferedMessage?.content === 'string' + ? bufferedMessage.content + : '', + frames: [], + localCalls: bufferedCalls.filter(isLocalCall), + reasoning: readReasoning(bufferedMessage), + remainingCalls: bufferedCalls.filter( + (toolCall) => !isLocalCall(toolCall), + ), + role: bufferedMessage?.role ?? 'assistant', + toolCalls: bufferedCalls, + usage, + }; + } else { + // Streamed rather than buffered: this is the iteration that very + // often ends the turn, and buffering it would make the user wait + // for the whole answer before seeing any of it. Text is forwarded + // as it arrives; only tool-call frames are held, since a server + // tool still has to be answered locally. + probe = await probeServerToolStream({ + canContinue: () => !cancelled, + context, + emitRaw: (frame) => + controller.enqueue(encoder.encode(`${frame}\n\n`)), + fetchProvider, + onReader: (reader) => { + activeReader = reader; + }, + response, + searchProvider, + }); + if (cancelled) return; + activeReader = null; + usage = sumUsage(usage, context.usage); + + // No server tool to answer, so the held frames — withheld only + // because they *might* have been one — are forwarded as-is. + if (!probe.localCalls.length) { + probe.frames.forEach((frame) => + controller.enqueue(encoder.encode(`${frame}\n\n`)), + ); + controller.close(); + return; + } } - usage = sumUsage(usage, payload.usage); - const message = payload.choices?.[0]?.message; - const calls = message?.tool_calls ?? []; - const nextLocalCalls = calls.filter( - (toolCall) => - (Boolean(searchProvider) && isWebSearchToolCall(toolCall)) || - (Boolean(fetchProvider) && isWebFetchToolCall(toolCall)), - ); - const nextRemainingCalls = calls.filter( - (toolCall) => - (!searchProvider || !isWebSearchToolCall(toolCall)) && - (!fetchProvider || !isWebFetchToolCall(toolCall)), - ); + // The model is going to search again, so anything it just said is + // part of the visible turn rather than a discarded step. A streamed + // iteration already forwarded it through `emitRaw`, so only a + // buffered one — whose payload never reached the client — needs it + // re-emitted here. + const iterationText = buffered ? probe.content.trim() : ''; + const iterationReasoning = buffered ? probe.reasoning.trim() : ''; - if (!nextLocalCalls.length) { - finalPayload = { ...payload, ...(usage ? { usage } : {}) }; - break; + if (iterationText) { + emitJson(controller, { + choices: [{ delta: { content: iterationText }, index: 0 }], + created: context.responseCreated, + id: responseId, + model: responseModel, + object: `${responseObject}.chunk`, + }); + } + + if (iterationReasoning) { + emitJson(controller, { + choices: [ + { delta: { reasoning_content: iterationReasoning }, index: 0 }, + ], + created: context.responseCreated, + id: responseId, + model: responseModel, + object: `${responseObject}.chunk`, + }); } + const message: ChatCompletionMessage = { + content: probe.content || null, + role: probe.role, + tool_calls: probe.toolCalls, + ...(probe.reasoning ? { reasoning_content: probe.reasoning } : {}), + }; + const nextLocalCalls = probe.localCalls; + const nextRemainingCalls = probe.remainingCalls; + const nextInvocations = nextLocalCalls.map((toolCall, index) => buildServerToolInvocation(toolCall, iteration, index), ); @@ -988,7 +1292,13 @@ const createInlineServerToolStream = async ({ if (nextRemainingCalls.length) { finalPayload = buildMixedTurnPayload({ message, - payload, + payload: { + choices: [{ message }], + created: context.responseCreated, + id: responseId, + model: responseModel, + object: responseObject, + }, remainingCalls: nextRemainingCalls, searchResults: nextResults.map((result) => result.content), usage, @@ -1019,22 +1329,74 @@ const createInlineServerToolStream = async ({ (tool) => !isWebSearchTool(tool) && !isWebFetchTool(tool), ), }, - 'buffer', + 'stream', ); - finalPayload = await readBufferedChatCompletionPayload(response); + const isEventStream = (response.headers.get('content-type') ?? '') + .toLowerCase() + .includes('text/event-stream'); - if (!response.ok || finalPayload.error) { - emitJson(controller, finalPayload as JsonRecord); - controller.enqueue(encoder.encode('data: [DONE]\n\n')); - controller.close(); - return; - } + if (!isEventStream) { + finalPayload = await readBufferedChatCompletionPayload(response); - usage = sumUsage(usage, finalPayload.usage); - finalPayload = { - ...finalPayload, - ...(usage ? { usage } : {}), - }; + if (!response.ok || finalPayload.error) { + emitJson(controller, finalPayload as JsonRecord); + controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.close(); + return; + } + + usage = sumUsage(usage, finalPayload.usage); + finalPayload = { ...finalPayload, ...(usage ? { usage } : {}) }; + } else { + const probe = await probeServerToolStream({ + canContinue: () => !cancelled, + context, + emitRaw: (frame) => + controller.enqueue(encoder.encode(`${frame}\n\n`)), + fetchProvider, + onReader: (reader) => { + activeReader = reader; + }, + response, + searchProvider, + }); + if (cancelled) return; + activeReader = null; + usage = sumUsage(usage, context.usage); + + // With every server tool stripped, a tool call here can only be a + // client-owned one; hand it back so the client resolves it. + if (probe.remainingCalls.length) { + const fallbackMessage: ChatCompletionMessage = { + content: probe.content || null, + role: probe.role, + tool_calls: probe.toolCalls, + ...(probe.reasoning + ? { reasoning_content: probe.reasoning } + : {}), + }; + + finalPayload = buildMixedTurnPayload({ + message: fallbackMessage, + payload: { + choices: [{ message: fallbackMessage }], + created: context.responseCreated, + id: responseId, + model: responseModel, + object: responseObject, + }, + remainingCalls: probe.remainingCalls, + searchResults: [], + usage, + }); + } else { + probe.frames.forEach((frame) => + controller.enqueue(encoder.encode(`${frame}\n\n`)), + ); + controller.close(); + return; + } + } } await pipeResponse( @@ -1134,6 +1496,11 @@ export const executeWebSearchLoop = async ({ let payload: ChatCompletionPayload | null = null; let usage: unknown = null; const executions: ServerToolExecution[] = []; + // Text and reasoning the model produced before a *later* server-tool call. + // Only the last iteration's message survives in `payload`, so a multi-hop + // turn has to carry its earlier steps forward explicitly. + const intermediateTexts: string[] = []; + const intermediateReasonings: string[] = []; const initialMode: ServerToolUpstreamMode = searchProvider && fetchProvider ? 'detect-both' @@ -1166,10 +1533,27 @@ export const executeWebSearchLoop = async ({ return { body: loopBody, executions, response }; } - payload = (await response.json()) as ChatCompletionPayload; + // The payload is only needed to detect a tool call or a failure, so read + // the body once and reuse it: the caller reads it again to build the + // client's answer, and a spent body would surface as a 500. + const buffered = await response.clone().text(); + + try { + payload = JSON.parse(buffered) as ChatCompletionPayload; + } catch (error) { + if (response.ok) { + throw error; + } + + payload = {}; + } if (!response.ok || payload.error) { - return { body: loopBody, executions, response }; + return { + body: loopBody, + executions, + response: await buildServerToolFailureResponse(response), + }; } usage = sumUsage(usage, payload.usage); @@ -1182,15 +1566,17 @@ export const executeWebSearchLoop = async ({ (Boolean(fetchProvider) && isWebFetchToolCall(toolCall)), ); const remainingCalls = toolCalls.filter( - (toolCall) => - (!searchProvider || !isWebSearchToolCall(toolCall)) && - (!fetchProvider || !isWebFetchToolCall(toolCall)), + (toolCall) => !localCalls.includes(toolCall), ); if (!localCalls.length) { break; } + const iterationText = + typeof message?.content === 'string' ? message.content.trim() : ''; + const iterationReasoning = readReasoning(message).trim(); + const invocations = localCalls.map( (toolCall, index): ServerToolInvocation => isWebFetchToolCall(toolCall) @@ -1254,7 +1640,14 @@ export const executeWebSearchLoop = async ({ executions, response: Response.json( buildMixedTurnPayload({ - message, + // `buildMixedTurnPayload` reads this iteration's text and reasoning + // off `message`, so only the earlier iterations go on top; the + // current one is folded in by the helper itself. + message: withIntermediateTurns({ + payload, + reasonings: intermediateReasonings, + texts: intermediateTexts, + }).choices?.[0]?.message, payload, remainingCalls, searchResults: results.map((result) => result.content), @@ -1265,6 +1658,16 @@ export const executeWebSearchLoop = async ({ }; } + // This iteration is complete and the loop continues, so its text becomes + // part of what the final answer has to carry. + if (iterationText) { + intermediateTexts.push(iterationText); + } + + if (iterationReasoning) { + intermediateReasonings.push(iterationReasoning); + } + messages.push(message as JsonRecord); messages.push( ...results.map((result) => ({ @@ -1298,18 +1701,45 @@ export const executeWebSearchLoop = async ({ }, 'buffer', ); - payload = (await finalResponse.json()) as ChatCompletionPayload; + // Cloned before the read so the failure path can replay the body verbatim + // rather than hand back a spent response the caller cannot read again. + const finalBuffered = await finalResponse.clone().text(); + + try { + payload = JSON.parse(finalBuffered) as ChatCompletionPayload; + } catch (error) { + // A malformed success payload is a real bug worth surfacing; a failure + // status already tells the caller everything, so its body is whatever + // the upstream sent, JSON or not. + if (finalResponse.ok) { + throw error; + } + + payload = {}; + } + usage = sumUsage(usage, payload.usage); if (!finalResponse.ok || payload.error) { - return { body: loopBody, executions, response: finalResponse }; + return { + body: loopBody, + executions, + response: await buildServerToolFailureResponse(finalResponse), + }; } return { body: loopBody, executions, response: Response.json( - { ...payload, ...(usage ? { usage } : {}) }, + { + ...withIntermediateTurns({ + payload, + reasonings: intermediateReasonings, + texts: intermediateTexts, + }), + ...(usage ? { usage } : {}), + }, { status: finalResponse.status }, ), }; @@ -1319,7 +1749,14 @@ export const executeWebSearchLoop = async ({ body: loopBody, executions, response: Response.json( - { ...payload, ...(usage ? { usage } : {}) }, + { + ...withIntermediateTurns({ + payload, + reasonings: intermediateReasonings, + texts: intermediateTexts, + }), + ...(usage ? { usage } : {}), + }, { status: response!.status }, ), }; diff --git a/tests/server/upstream-error-response.test.ts b/tests/server/upstream-error-response.test.ts new file mode 100644 index 0000000..59e9fad --- /dev/null +++ b/tests/server/upstream-error-response.test.ts @@ -0,0 +1,204 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { NextRequest } from 'next/server'; + +import { updateSettings } from '@/lib/server/domain/config'; +import { isDebugEnabled, updateDebugSettings } from '@/lib/server/domain/debug'; +import { resetWebSearchProviders } from '@/lib/server/search'; +import { handleMessagesRequest } from '@/lib/server/proxy/anthropic'; +import { proxyChatCompletions } from '@/lib/server/proxy/codebuddy'; +import { + addCredential, + resetCredentialRuntimeState, +} from '@/lib/server/domain/credentials'; +import { resetUsageStats } from '@/lib/server/domain/stats'; +import { resetStorageRuntime } from '@/lib/server/storage'; + +/** + * An upstream error response has to survive being inspected more than once. + * + * The server-tool loop reads the body to find out whether the model asked for a + * search, `logUpstreamFailure` reads it to record what went wrong, and the + * route layer reads it again to build the answer the client actually sees. A + * `Response` body can only be consumed once, so those reads have to share: + * every one of them after the first used to fail with + * `Body already used`, which surfaced to clients as a 500 instead of the real + * upstream status. + */ + +const tempRootDir = path.join(process.cwd(), '.tmp-test-upstream-error-root'); + +const cleanupTempState = (): void => { + fs.rmSync(tempRootDir, { force: true, recursive: true }); +}; + +const makeNextRequest = (): NextRequest => + new NextRequest('http://localhost/v1/chat/completions', { method: 'POST' }); + +const makeJsonResponse = ( + payload: Record, + status = 200, +): Response => + new Response(JSON.stringify(payload), { + status, + headers: { 'Content-Type': 'application/json' }, + }); + +describe('upstream error responses', () => { + beforeEach(async () => { + cleanupTempState(); + vi.restoreAllMocks(); + vi.clearAllMocks(); + vi.spyOn(process, 'cwd').mockReturnValue(tempRootDir); + process.env.CODEBUDDY_AUTH_MODE = 'auto'; + process.env.CODEBUDDY_API_KEY = ''; + resetStorageRuntime(); + resetCredentialRuntimeState(); + await resetUsageStats(); + addCredential({ + bearer_token: 'error-path-token', + user_id: 'error-path@example.com', + }); + }); + + afterEach(() => { + cleanupTempState(); + }); + + it('keeps the body readable after the failure has been logged', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeJsonResponse( + { + code: 6004, + msg: '您的使用量已超出频率限制', + }, + 429, + ), + ); + + const response = await proxyChatCompletions( + makeNextRequest(), + { + model: 'glm-5.1', + messages: [{ role: 'user', content: 'hello' }], + }, + undefined, + undefined, + ); + + expect(response.status).toBe(429); + await expect(response.json()).resolves.toMatchObject({ + error: { detail: expect.stringContaining('6004') }, + }); + }); + + it('surfaces the upstream status through the server-tool loop', async () => { + // Reproduces the reported failure: a rate-limited upstream answering a + // request that declared a server web tool. The loop reads the body to + // decide whether the model asked for a search, and the route layer reads + // it again to build the client's answer. + process.env.SEARXNG_URL = 'https://searx.test'; + resetWebSearchProviders(); + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'searxng' }); + + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeJsonResponse( + { + code: 6004, + msg: '您的使用量已超出频率限制,将在 2026-09-18 11:35:06 UTC+8 重置', + requestId: '5d3bd1789bec4171a6dbc94664e680a3', + }, + 429, + ), + ); + + const response = await proxyChatCompletions( + makeNextRequest(), + { + model: 'glm-5.1', + messages: [{ role: 'user', content: 'hello' }], + stream: true, + tools: [{ type: 'web_search_20260209', name: 'web_search' }], + }, + undefined, + undefined, + ); + + expect(response.status).toBe(429); + await expect(response.text()).resolves.toContain('6004'); + }); + + it('keeps the body readable when debug tracing snapshots it', async () => { + await updateDebugSettings({ enabled: true }); + expect(await isDebugEnabled()).toBe(true); + + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeJsonResponse( + { + code: 6004, + msg: '您的使用量已超出频率限制', + }, + 429, + ), + ); + + const response = await proxyChatCompletions( + makeNextRequest(), + { + model: 'glm-5.1', + messages: [{ role: 'user', content: 'hello' }], + }, + undefined, + undefined, + ); + + expect(response.status).toBe(429); + await expect(response.json()).resolves.toMatchObject({ + error: { detail: expect.stringContaining('6004') }, + }); + }); + + it.each([false, true])( + 'reports the upstream rate limit to an Anthropic client (stream: %s)', + async (stream) => { + // Claude Code reads `rate_limit_error` to decide whether to back off, so + // the upstream status has to survive the server-tool bridge on both the + // non-streaming and the streaming path. + process.env.SEARXNG_URL = 'https://searx.test'; + resetWebSearchProviders(); + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'searxng' }); + + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + makeJsonResponse( + { + code: 6004, + msg: '您的使用量已超出频率限制,将在 2026-09-18 11:35:06 UTC+8 重置', + requestId: '5d3bd1789bec4171a6dbc94664e680a3', + }, + 429, + ), + ); + + const response = await handleMessagesRequest( + new NextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ content: 'hello', role: 'user' }], + model: 'claude-sonnet-4.6', + stream, + tools: [{ name: 'web_search', type: 'web_search_20260209' }], + } as never, + ); + + const body = await response.text(); + + if (!stream) { + expect(response.status).toBe(429); + } + + expect(body).toContain('rate_limit_error'); + expect(body).toContain('6004'); + }, + ); +}); diff --git a/tests/server/web-search.test.ts b/tests/server/web-search.test.ts index 3606924..d25f194 100644 --- a/tests/server/web-search.test.ts +++ b/tests/server/web-search.test.ts @@ -1845,6 +1845,137 @@ describe('server local web search', () => { tools: [{ type: 'web_search_preview' }], }); + it('hands a client call from a buffered iteration back to the client', async () => { + await enableSearxngSearch(); + let call = 0; + const upstream = vi.fn(async () => { + call += 1; + + if (call === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + function: { + arguments: '{"query":"mixed"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + // A buffered iteration can mix a server tool with a client-owned one: + // the server tool runs here, the client's is handed back unanswered. + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + content: 'Checking both.', + tool_calls: [ + { + function: { + arguments: '{"query":"second"}', + name: 'web_search', + }, + }, + { + id: 'call_client', + function: { arguments: '{}', name: 'client_tool' }, + type: 'function', + }, + ], + }, + }, + ], + }); + }); + + const result = await executeWebSearchLoop({ + body: inlineBody(), + callbacks: { emitStreamEvents: true }, + callUpstream: upstream, + }); + const text = await result!.response!.text(); + + expect(text).toContain('Checking both.'); + expect(text).toContain('client_tool'); + expect(text).toContain('"finish_reason":"tool_calls"'); + }); + + it('ends the turn when a follow-up iteration stops calling server tools', async () => { + await enableSearxngSearch(); + let call = 0; + const upstream = vi.fn(async () => { + call += 1; + + if (call === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_search', + index: 0, + function: { + arguments: '{"query":"only hop"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + // No server tool this time: the held frames are forwarded untouched + // and the turn is over. + return makeSseResponse( + { + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_client', + index: 0, + function: { arguments: '{}', name: 'client_tool' }, + type: 'function', + }, + ], + }, + index: 0, + }, + ], + }, + { choices: [{ delta: {}, finish_reason: 'tool_calls', index: 0 }] }, + ); + }); + + const result = await executeWebSearchLoop({ + body: inlineBody(), + callbacks: { emitStreamEvents: true }, + callUpstream: upstream, + }); + const text = await result!.response!.text(); + + expect(text).toContain('client_tool'); + expect(text).toContain('"finish_reason":"tool_calls"'); + expect(upstream).toHaveBeenCalledTimes(2); + }); + it('keeps malformed frames and returns mixed client tool calls', async () => { await enableSearxngSearch(); const upstream = vi.fn(async () => { @@ -2242,7 +2373,7 @@ describe('server local web search', () => { expect(upstream).toHaveBeenCalledTimes(6); }); - it('returns an error when the final budget fallback fails upstream', async () => { + it('streams the budget fallback answer once local tools are dropped', async () => { await enableSearxngSearch(); let call = 0; const upstream = vi.fn(async (body) => { @@ -2255,10 +2386,8 @@ describe('server local web search', () => { delta: { tool_calls: [ { - id: 'call_initial', - index: 0, function: { - arguments: '{"query":"fallback error"}', + arguments: '{"query":"loop 0"}', name: 'web_search', }, }, @@ -2279,9 +2408,8 @@ describe('server local web search', () => { message: { tool_calls: [ { - id: `call_${call}`, function: { - arguments: '{"query":"again"}', + arguments: `{"query":"loop ${call - 1}"}`, name: 'web_search', }, }, @@ -2292,7 +2420,12 @@ describe('server local web search', () => { }); } - return makeJsonResponse({ error: { message: 'fallback failed' } }, 502); + // The budget is spent and every server tool is gone, so this answer + // ends the turn. It is streamed, so it must reach the client as-is + // rather than being buffered into a payload. + return makeSseResponse({ + choices: [{ delta: { content: 'Budget stream.' }, index: 0 }], + }); }); const result = await executeWebSearchLoop({ @@ -2300,83 +2433,399 @@ describe('server local web search', () => { callbacks: { emitStreamEvents: true }, callUpstream: upstream, }); + const text = await result!.response!.text(); + const finalBody = upstream.mock.calls.at(-1)?.[0]; - await expect(result!.response!.text()).resolves.toContain( - 'fallback failed', - ); + expect(text).toContain('Budget stream.'); + expect(finalBody?.tools).toEqual([]); expect(upstream).toHaveBeenCalledTimes(6); }); - it('leaves an empty non-SSE initial response untouched', async () => { + it('uses a JSON budget fallback answer when upstream does not stream', async () => { await enableSearxngSearch(); - const response = new Response(null, { status: 204 }); + let call = 0; + const upstream = vi.fn(async (body) => { + call += 1; + + if (call === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + function: { + arguments: '{"query":"loop 0"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + if (body.tools?.length) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + tool_calls: [ + { + function: { + arguments: `{"query":"loop ${call - 1}"}`, + name: 'web_search', + }, + }, + ], + }, + }, + ], + usage: { total_tokens: 1 }, + }); + } + + // A non-streaming upstream still has to produce a usable answer once + // the budget is spent. + return makeJsonResponse({ + choices: [ + { finish_reason: 'stop', message: { content: 'JSON fallback.' } }, + ], + usage: { total_tokens: 4 }, + }); + }); const result = await executeWebSearchLoop({ body: inlineBody(), callbacks: { emitStreamEvents: true }, - callUpstream: async () => response, + callUpstream: upstream, }); + const text = await result!.response!.text(); - expect(result?.response).toBe(response); - await expect(result!.response!.text()).resolves.toBe(''); + expect(text).toContain('JSON fallback.'); + // Usage accumulates across every iteration of the loop. + expect(text).toContain('"total_tokens":8'); + expect(upstream).toHaveBeenCalledTimes(6); }); - it('always asks upstream to stream and buffers the response', async () => { - process.env.SEARXNG_URL = 'https://searx.test'; - resetWebSearchProviders(); - await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'searxng' }); - - const upstreamBodies: Array> = []; + it('hands a client tool call from the budget fallback back to the client', async () => { + await enableSearxngSearch(); let call = 0; + const upstream = vi.fn(async (body) => { + call += 1; - const fetchMock = vi.fn( - async (input: RequestInfo | URL, init?: RequestInit) => { - const url = String(input); - - if (url.includes('searx.test')) { - return makeJsonResponse({ - results: [{ content: 'snip', title: 'T', url: 'https://t.test' }], - }); - } - - call += 1; - upstreamBodies.push(JSON.parse(String(init?.body ?? '{}'))); - - // Upstream only ever speaks SSE. - const chunk = - call === 1 - ? { - choices: [ + if (call === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ { - delta: { - tool_calls: [ - { - id: 'call_1', - index: 0, - type: 'function', - function: { - arguments: '{"query":"q1"}', - name: 'web_search', - }, - }, - ], + function: { + arguments: '{"query":"loop 0"}', + name: 'web_search', }, - finish_reason: 'tool_calls', }, ], - } - : { - choices: [ - { delta: { content: 'Done.' }, finish_reason: 'stop' }, - ], - }; - - return new Response( - `data: ${JSON.stringify(chunk)}\n\ndata: [DONE]\n\n`, - { - status: 200, - headers: { - 'Content-Type': 'text/event-stream; charset=utf-8', + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + if (body.tools?.length) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + tool_calls: [ + { + function: { + arguments: `{"query":"loop ${call - 1}"}`, + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + // Every server tool was stripped, so a tool call here can only be the + // client's own: it has to be handed back rather than executed. + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_client', + index: 0, + function: { arguments: '{}', name: 'client_tool' }, + type: 'function', + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + }); + + const result = await executeWebSearchLoop({ + body: inlineBody(), + callbacks: { emitStreamEvents: true }, + callUpstream: upstream, + }); + const text = await result!.response!.text(); + + expect(text).toContain('client_tool'); + expect(text).toContain('"finish_reason":"tool_calls"'); + expect(upstream).toHaveBeenCalledTimes(6); + }); + + it('stops the budget fallback when the client disconnects', async () => { + await enableSearxngSearch(); + const encoder = new TextEncoder(); + let call = 0; + let resolveCancel: (() => void) | undefined; + let upstreamCancelled: Promise | undefined; + + const upstream = vi.fn(async (body) => { + call += 1; + + if (call === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + function: { + arguments: '{"query":"loop 0"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + if (body.tools?.length) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + tool_calls: [ + { + function: { + arguments: `{"query":"loop ${call - 1}"}`, + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + // The fallback answer never finishes, so only a client-side + // cancellation can end this turn. + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'Fallback.' }, index: 0 }], + })}\n\n`, + ), + ); + upstreamCancelled = new Promise((resolve) => { + resolveCancel = () => resolve(true); + }); + }, + cancel: () => { + resolveCancel?.(); + }, + }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + }); + + const result = await executeWebSearchLoop({ + body: inlineBody(), + callbacks: { emitStreamEvents: true }, + callUpstream: upstream, + }); + const reader = result!.response!.body!.getReader(); + const decoder = new TextDecoder(); + let seen = ''; + + while (!seen.includes('Fallback.')) { + const chunk = await reader.read(); + expect(chunk.done).toBe(false); + seen += decoder.decode(chunk.value); + } + + await reader.cancel(); + + // The disconnect has to reach the parked upstream read, not just the + // downstream stream: a stalled upstream would otherwise stay alive. + await expect(upstreamCancelled).resolves.toBe(true); + expect(upstream).toHaveBeenCalledTimes(6); + }); + + it('returns an error when the final budget fallback fails upstream', async () => { + await enableSearxngSearch(); + let call = 0; + const upstream = vi.fn(async (body) => { + call += 1; + + if (call === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_initial', + index: 0, + function: { + arguments: '{"query":"fallback error"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + if (body.tools?.length) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + tool_calls: [ + { + id: `call_${call}`, + function: { + arguments: '{"query":"again"}', + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + return makeJsonResponse({ error: { message: 'fallback failed' } }, 502); + }); + + const result = await executeWebSearchLoop({ + body: inlineBody(), + callbacks: { emitStreamEvents: true }, + callUpstream: upstream, + }); + + await expect(result!.response!.text()).resolves.toContain( + 'fallback failed', + ); + expect(upstream).toHaveBeenCalledTimes(6); + }); + + it('leaves an empty non-SSE initial response untouched', async () => { + await enableSearxngSearch(); + const response = new Response(null, { status: 204 }); + + const result = await executeWebSearchLoop({ + body: inlineBody(), + callbacks: { emitStreamEvents: true }, + callUpstream: async () => response, + }); + + expect(result?.response).toBe(response); + await expect(result!.response!.text()).resolves.toBe(''); + }); + + it('always asks upstream to stream and buffers the response', async () => { + process.env.SEARXNG_URL = 'https://searx.test'; + resetWebSearchProviders(); + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'searxng' }); + + const upstreamBodies: Array> = []; + let call = 0; + + const fetchMock = vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + + if (url.includes('searx.test')) { + return makeJsonResponse({ + results: [{ content: 'snip', title: 'T', url: 'https://t.test' }], + }); + } + + call += 1; + upstreamBodies.push(JSON.parse(String(init?.body ?? '{}'))); + + // Upstream only ever speaks SSE. + const chunk = + call === 1 + ? { + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_1', + index: 0, + type: 'function', + function: { + arguments: '{"query":"q1"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + } + : { + choices: [ + { delta: { content: 'Done.' }, finish_reason: 'stop' }, + ], + }; + + return new Response( + `data: ${JSON.stringify(chunk)}\n\ndata: [DONE]\n\n`, + { + status: 200, + headers: { + 'Content-Type': 'text/event-stream; charset=utf-8', }, }, ); @@ -3973,8 +4422,371 @@ describe('chat proxy web search integration', () => { expect(secondMessages).toContainEqual( expect.objectContaining({ role: 'tool', tool_call_id: 'call_search' }), ); - expect(searchCalls).toBe(1); - expect(upstreamCalls).toBe(2); + expect(searchCalls).toBe(1); + expect(upstreamCalls).toBe(2); + }); + + it('keeps the text a multi-hop turn wrote between two searches', async () => { + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'codebuddy' }); + let upstreamCalls = 0; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/search')) { + return makeJsonResponse({ + results: [ + { snippet: 'snip', title: 'Result', url: 'https://r.test' }, + ], + }); + } + + upstreamCalls++; + + if (upstreamCalls === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_first', + index: 0, + function: { + arguments: '{"query":"first hop"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + // The model speaks and reasons before searching again. That text is part + // of the visible turn, so it must not be swallowed by the loop. + if (upstreamCalls === 2) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + content: 'First hop was inconclusive.', + reasoning_content: 'Narrowing the query.', + tool_calls: [ + { + id: 'call_second', + function: { + arguments: '{"query":"second hop"}', + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + return makeJsonResponse({ + choices: [ + { + finish_reason: 'stop', + message: { content: 'Two hops later.' }, + }, + ], + }); + }); + + const response = await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Two hop question' }], + stream: true, + tools: [ + { + type: 'web_search_20260209', + name: 'web_search', + input_schema: {}, + }, + ], + }, + ); + const text = await response.text(); + + expect(text).toContain('First hop was inconclusive.'); + expect(text).toContain('Narrowing the query.'); + expect(text).toContain('Two hops later.'); + expect(text.indexOf('First hop was inconclusive.')).toBeLessThan( + text.indexOf('Two hops later.'), + ); + expect((text.match(/"type":"server_tool_use"/g) ?? []).length).toBe(2); + expect((text.match(/"type":"web_search_tool_result"/g) ?? []).length).toBe( + 2, + ); + // A buffered iteration is re-emitted from the payload, so it must appear + // exactly once — not once from the payload and once from the fold. + expect((text.match(/First hop was inconclusive\./g) ?? []).length).toBe(1); + expect((text.match(/Narrowing the query\./g) ?? []).length).toBe(1); + expect(upstreamCalls).toBe(3); + }); + + it('does not repeat text a streamed iteration already forwarded', async () => { + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'codebuddy' }); + const encoder = new TextEncoder(); + let upstreamCalls = 0; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/search')) { + return makeJsonResponse({ + results: [ + { snippet: 'snip', title: 'Result', url: 'https://r.test' }, + ], + }); + } + + upstreamCalls++; + + if (upstreamCalls === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_first', + index: 0, + function: { + arguments: '{"query":"first hop"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + if (upstreamCalls === 2) { + // A streamed iteration: its deltas are forwarded as they arrive, so + // re-emitting the accumulated text afterwards would duplicate it. + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [ + { delta: { content: 'Spoken between hops.' }, index: 0 }, + ], + })}\n\n`, + ), + ); + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [ + { + delta: { reasoning_content: 'Thinking between hops.' }, + index: 0, + }, + ], + })}\n\n`, + ), + ); + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_second', + index: 0, + function: { + arguments: '{"query":"second hop"}', + name: 'web_search', + }, + }, + ], + }, + index: 0, + }, + ], + })}\n\n`, + ), + ); + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [ + { delta: {}, finish_reason: 'tool_calls', index: 0 }, + ], + })}\n\n`, + ), + ); + controller.close(); + }, + }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + } + + return makeJsonResponse({ + choices: [ + { finish_reason: 'stop', message: { content: 'Two hops later.' } }, + ], + }); + }); + + const response = await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Two hop question' }], + stream: true, + tools: [ + { + type: 'web_search_20260209', + name: 'web_search', + input_schema: {}, + }, + ], + }, + ); + const text = await response.text(); + + expect(text).toContain('Spoken between hops.'); + expect(text).toContain('Thinking between hops.'); + expect(text).toContain('Two hops later.'); + expect((text.match(/Spoken between hops\./g) ?? []).length).toBe(1); + expect((text.match(/Thinking between hops\./g) ?? []).length).toBe(1); + expect(upstreamCalls).toBe(3); + }); + + it('streams the final Messages answer as upstream produces it', async () => { + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'codebuddy' }); + const encoder = new TextEncoder(); + let upstreamCalls = 0; + let releaseSecondChunk: (() => void) | undefined; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/search')) { + return makeJsonResponse({ + results: [ + { snippet: 'snip', title: 'Result', url: 'https://r.test' }, + ], + }); + } + + upstreamCalls++; + + if (upstreamCalls === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_search', + index: 0, + function: { + arguments: '{"query":"streamed answer"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + // The final answer arrives in two pieces, the second only after the test + // releases it — so a buffered replay collapses both into one instant. + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'First half.' }, index: 0 }], + })}\n\n`, + ), + ); + releaseSecondChunk = () => { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [ + { delta: { content: ' Second half.' }, index: 0 }, + ], + })}\n\n`, + ), + ); + controller.close(); + }; + }, + }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + }); + + const response = await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Stream the answer' }], + stream: true, + tools: [ + { + type: 'web_search_20260209', + name: 'web_search', + input_schema: {}, + }, + ], + }, + ); + + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let before = ''; + + while (!before.includes('First half.')) { + const chunk = await reader.read(); + expect(chunk.done).toBe(false); + before += decoder.decode(chunk.value); + } + + // The first half has to arrive before the second is even produced; a + // buffered final iteration would withhold it until the whole answer was in. + expect(before).not.toContain('Second half.'); + expect(releaseSecondChunk).toBeTypeOf('function'); + releaseSecondChunk!(); + + let remainder = ''; + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + remainder += decoder.decode(chunk.value); + } + + expect(remainder).toContain('Second half.'); + expect(before).toContain('"type":"web_search_tool_result"'); }); it('streams Messages server_tool_use before the backend result', async () => { @@ -4119,6 +4931,107 @@ describe('chat proxy web search integration', () => { }, ); + it('stops the follow-up iteration when the client disconnects mid-answer', async () => { + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'codebuddy' }); + const encoder = new TextEncoder(); + let upstreamCalls = 0; + let resolveCancel: (() => void) | undefined; + let upstreamCancelled: Promise | undefined; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/search')) { + return makeJsonResponse({ + results: [ + { snippet: 'snip', title: 'Result', url: 'https://r.test' }, + ], + }); + } + + upstreamCalls++; + + if (upstreamCalls === 1) { + return makeSseResponse({ + choices: [ + { + delta: { + tool_calls: [ + { + id: 'call_search', + index: 0, + function: { + arguments: '{"query":"cancel mid answer"}', + name: 'web_search', + }, + }, + ], + }, + finish_reason: 'tool_calls', + index: 0, + }, + ], + }); + } + + // The answer never finishes on its own, so only a cancellation can end + // this iteration. + return new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + `data: ${JSON.stringify({ + choices: [{ delta: { content: 'Partial.' }, index: 0 }], + })}\n\n`, + ), + ); + upstreamCancelled = new Promise((resolve) => { + resolveCancel = () => resolve(true); + }); + }, + cancel: () => { + resolveCancel?.(); + }, + }), + { headers: { 'Content-Type': 'text/event-stream' } }, + ); + }); + + const response = await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Cancel mid answer' }], + stream: true, + tools: [ + { + type: 'web_search_20260209', + name: 'web_search', + input_schema: {}, + }, + ], + }, + ); + + const reader = response.body!.getReader(); + const decoder = new TextDecoder(); + let seen = ''; + + while (!seen.includes('Partial.')) { + const chunk = await reader.read(); + expect(chunk.done).toBe(false); + seen += decoder.decode(chunk.value); + } + + await reader.cancel(); + + // The disconnect must reach the parked upstream read, not only the + // downstream stream, so a stalled upstream does not stay alive. + await expect(upstreamCancelled).resolves.toBe(true); + expect(upstreamCalls).toBe(2); + }); + it('cancels a late Messages upstream stream after disconnect', async () => { let finishSearch: ((response: Response) => void) | undefined; let upstreamCalls = 0; @@ -4183,6 +5096,203 @@ describe('chat proxy web search integration', () => { expect(upstreamCalls).toBe(1); }); + it('folds multi-hop text into a non-streaming Messages answer', async () => { + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'codebuddy' }); + let upstreamCalls = 0; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/search')) { + return makeJsonResponse({ + results: [ + { snippet: 'snip', title: 'Result', url: 'https://r.test' }, + ], + }); + } + + upstreamCalls++; + + if (upstreamCalls === 1) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + tool_calls: [ + { + id: 'call_first', + function: { + arguments: '{"query":"first hop"}', + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + if (upstreamCalls === 2) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + content: 'First hop was inconclusive.', + reasoning_content: 'Narrowing the query.', + tool_calls: [ + { + id: 'call_second', + function: { + arguments: '{"query":"second hop"}', + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + return makeJsonResponse({ + choices: [ + { finish_reason: 'stop', message: { content: 'Two hops later.' } }, + ], + }); + }); + + const response = await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Two hop question' }], + tools: [ + { + type: 'web_search_20260209', + name: 'web_search', + input_schema: {}, + }, + ], + }, + ); + const payload = (await response.json()) as { + content: Array<{ text?: string; thinking?: string; type: string }>; + }; + const text = payload.content + .filter((block) => block.type === 'text') + .map((block) => block.text ?? '') + .join(''); + const thinking = payload.content + .filter((block) => block.type === 'thinking') + .map((block) => block.thinking ?? '') + .join(''); + + // A non-streaming turn has no place to emit intermediate deltas, so the + // text is folded in ahead of the final answer rather than dropped. + expect(text).toContain('First hop was inconclusive.'); + expect(text).toContain('Two hops later.'); + expect(text.indexOf('First hop was inconclusive.')).toBeLessThan( + text.indexOf('Two hops later.'), + ); + expect(thinking).toContain('Narrowing the query.'); + expect(upstreamCalls).toBe(3); + }); + + it('does not repeat the current text in a non-streaming mixed turn', async () => { + await updateSettings({ CODEBUDDY_WEB_SEARCH_BACKEND: 'codebuddy' }); + let upstreamCalls = 0; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/search')) { + return makeJsonResponse({ + results: [ + { snippet: 'snip', title: 'Result', url: 'https://r.test' }, + ], + }); + } + + upstreamCalls++; + + if (upstreamCalls === 1) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + tool_calls: [ + { + id: 'call_first', + function: { + arguments: '{"query":"first hop"}', + name: 'web_search', + }, + }, + ], + }, + }, + ], + }); + } + + // A turn that carries its own text, asks for another search, and also + // calls a client-owned tool: the mixed payload already includes the + // current text, so folding it in again would duplicate it. + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + content: 'Checking both.', + reasoning_content: 'Weighing the results.', + tool_calls: [ + { + id: 'call_second', + function: { + arguments: '{"query":"second hop"}', + name: 'web_search', + }, + }, + { + id: 'call_client', + function: { arguments: '{}', name: 'client_tool' }, + type: 'function', + }, + ], + }, + }, + ], + }); + }); + + const response = await handleMessagesRequest( + makeNextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Mixed turn' }], + tools: [ + { + type: 'web_search_20260209', + name: 'web_search', + input_schema: {}, + }, + ], + }, + ); + const payload = (await response.json()) as { + content: Array<{ text?: string; thinking?: string; type: string }>; + }; + const serialized = JSON.stringify(payload); + + expect((serialized.match(/Checking both\./g) ?? []).length).toBe(1); + expect((serialized.match(/Weighing the results\./g) ?? []).length).toBe(1); + expect(upstreamCalls).toBe(2); + }); + it('maps a completed fetch to a Responses open_page call', async () => { await updateSettings({ CODEBUDDY_WEB_FETCH_BACKEND: 'codebuddy' }); let upstreamCalls = 0;