diff --git a/lib/server/proxy/anthropic.ts b/lib/server/proxy/anthropic.ts index b88c9de..eda62ac 100644 --- a/lib/server/proxy/anthropic.ts +++ b/lib/server/proxy/anthropic.ts @@ -12,6 +12,7 @@ import { getServerToolStreamEvent, getServerToolExecutions, type ServerToolExecution, + type ServerToolTurn, } from './web-search-loop'; import { anthropicStreamErrorChunks, @@ -128,6 +129,12 @@ interface OpenAIChatResponse { id?: string; model?: string; choices?: OpenAIChatChoice[]; + /** + * Per-hop grouping emitted by the local server-tool loop. Not part of the + * OpenAI protocol — it survives only as far as this file, which turns it into + * Anthropic content blocks. + */ + turns?: ServerToolTurn[]; usage?: OpenAIUsage; } @@ -601,79 +608,127 @@ const encodeOpaqueServerToolContent = (value: unknown): string => { }; const buildAnthropicServerToolBlocks = ( - executions: ServerToolExecution[], -): AnthropicContentBlock[] => - executions.flatMap((execution) => { - const id = createAnthropicId('srvtoolu'); - const result = - execution.type === 'web_search' - ? { - type: 'web_search_tool_result', - tool_use_id: id, - content: execution.result.results.map((item) => ({ - type: 'web_search_result', - url: item.url ?? '', - title: item.title ?? '', - encrypted_content: encodeOpaqueServerToolContent(item), - })), - } - : { - type: 'web_fetch_tool_result', - tool_use_id: id, + execution: ServerToolExecution, +): AnthropicContentBlock[] => { + const id = createAnthropicId('srvtoolu'); + const result = + execution.type === 'web_search' + ? { + type: 'web_search_tool_result', + tool_use_id: id, + content: execution.result.results.map((item) => ({ + type: 'web_search_result', + url: item.url ?? '', + title: item.title ?? '', + encrypted_content: encodeOpaqueServerToolContent(item), + })), + } + : { + type: 'web_fetch_tool_result', + tool_use_id: id, + content: { + type: 'web_fetch_result', + url: execution.result.url ?? execution.input.url, content: { - type: 'web_fetch_result', - url: execution.result.url ?? execution.input.url, - content: { - type: 'document', - source: { - type: 'text', - media_type: 'text/plain', - data: execution.result.content, - }, + type: 'document', + source: { + type: 'text', + media_type: 'text/plain', + data: execution.result.content, }, }, - }; - - return [ - { - type: 'server_tool_use', - id, - name: execution.type, - input: execution.input, - }, - result, - ]; + }, + }; + + return [ + { + type: 'server_tool_use', + id, + name: execution.type, + input: execution.input, + }, + result, + ]; +}; + +const buildAllAnthropicServerToolBlocks = ( + executions: ServerToolExecution[], +): AnthropicContentBlock[] => + executions.flatMap(buildAnthropicServerToolBlocks); + +/** + * Lays a server-tool turn out the way Anthropic does: each hop contributes its + * own thinking and text, followed by the tool blocks that hop triggered. + * + * `turns` carries the per-hop grouping the OpenAI-shaped payload cannot. Under + * that protocol a multi-hop turn collapses into one `content` string and one + * `reasoning_content` string, which loses where one hop's reasoning ends and the + * next begins — so the grouping has to be recovered before it is joined, which + * is why the loop emits it alongside the strings rather than this file + * reconstructing it. + * + * Anthropic's own server tools run multiple hops inside one assistant message, + * and a client replaying that message expects `[thinking] [text] [tool_use] + * [tool_result] [thinking] [text]`. Gathering the blocks by kind instead — every + * tool ahead of all the prose — puts each search before the reasoning that asked + * for it and merges hops that were never contiguous. + */ +const buildAnthropicTurnBlocks = ( + turns: ServerToolTurn[], +): AnthropicContentBlock[] => { + const blocks: AnthropicContentBlock[] = []; + + turns.forEach((turn) => { + if (turn.reasoning) { + blocks.push({ type: 'thinking', thinking: turn.reasoning }); + } + + if (turn.text) { + blocks.push({ type: 'text', text: turn.text }); + } + + blocks.push(...buildAllAnthropicServerToolBlocks(turn.executions)); }); + return blocks; +}; + const mapOpenAIResponseToAnthropic = ( openaiResponse: OpenAIChatResponse, model: string, serverToolExecutions: ServerToolExecution[] = [], + turns?: ServerToolTurn[], ): Record => { const choice = openaiResponse.choices?.[0]; const message = choice?.message; - const contentBlocks: AnthropicContentBlock[] = - buildAnthropicServerToolBlocks(serverToolExecutions); // Thinking / reasoning content const reasoningText = message?.reasoning_content ?? message?.reasoning ?? ''; - if (reasoningText) { - contentBlocks.push({ - type: 'thinking', - thinking: reasoningText, - }); - } - // Text content const textContent = typeof message?.content === 'string' ? message.content : ''; - if (textContent) { - contentBlocks.push({ - type: 'text', - text: textContent, - }); + // With per-hop grouping the turns already hold every block in order, prose + // included. Without it — no server tool ran, or a path that never grouped the + // hops — fall back to Anthropic's own order: thinking and text first, then + // the server-tool blocks they led to. + const contentBlocks: AnthropicContentBlock[] = turns + ? buildAnthropicTurnBlocks(turns) + : []; + + if (!turns) { + if (reasoningText) { + contentBlocks.push({ type: 'thinking', thinking: reasoningText }); + } + + if (textContent) { + contentBlocks.push({ type: 'text', text: textContent }); + } + + contentBlocks.push( + ...buildAllAnthropicServerToolBlocks(serverToolExecutions), + ); } // Tool calls @@ -1118,9 +1173,9 @@ const mapOpenAIStreamToAnthropicSSE = ( closeOpenTextBlocks(); serverToolExecutions.push(serverToolEvent.execution); const index = contentBlockCount++; - const resultBlock = buildAnthropicServerToolBlocks([ + const resultBlock = buildAnthropicServerToolBlocks( serverToolEvent.execution, - ])[1]; + )[1]; enqueueEvent({ type: 'content_block_start', index, @@ -1419,7 +1474,12 @@ export const handleMessagesRequest = async ( const payload = (await upstreamResponse.json()) as OpenAIChatResponse; return Response.json( - mapOpenAIResponseToAnthropic(payload, model, serverToolExecutions), + mapOpenAIResponseToAnthropic( + payload, + model, + serverToolExecutions, + payload.turns, + ), ); } catch (error) { return createAnthropicError( diff --git a/lib/server/proxy/web-search-loop.ts b/lib/server/proxy/web-search-loop.ts index 981db34..8336ce3 100644 --- a/lib/server/proxy/web-search-loop.ts +++ b/lib/server/proxy/web-search-loop.ts @@ -82,6 +82,15 @@ export interface ChatCompletionPayload { id?: string; model?: string; object?: string; + /** + * One entry per server-tool hop, in the order the model produced them. + * + * Carries the grouping `message.content` / `reasoning_content` cannot: a + * multi-hop turn joins every hop into one string per kind, which loses where + * one hop's reasoning ends and the next begins. Absent when no hop ran, so + * callers fall back to the OpenAI-shaped fields. + */ + turns?: ServerToolTurn[]; usage?: unknown; } @@ -474,6 +483,35 @@ const readReasoning = (message: ChatCompletionMessage | undefined): string => { return typeof message.reasoning === 'string' ? message.reasoning : ''; }; +/** + * Pairs each hop's reasoning and text with the calls that hop made. + * + * The three arrays are index-aligned — entry N is hop N — so zipping them back + * together is what restores the grouping a joined string cannot express. `texts` + * and `reasonings` are one entry longer than `executions`, because the closing + * hop answers instead of calling another tool. + */ +const buildIntermediateTurns = ({ + executions, + reasonings, + texts, +}: { + executions: ServerToolExecution[][]; + reasonings: string[]; + texts: string[]; +}): ServerToolTurn[] => + Array.from( + { length: Math.max(reasonings.length, texts.length) }, + (_, index) => ({ + // Only `executions` can run short: the closing hop answers without + // calling anything, so it has an entry in the prose arrays but none here. + // The three arrays stay aligned because every hop appends to all of them. + executions: executions[index] ?? [], + reasoning: reasonings[index], + text: texts[index], + }), + ); + /** * Folds the text a multi-hop turn produced before its later server-tool calls * into the payload the client receives. @@ -482,12 +520,17 @@ const readReasoning = (message: ChatCompletionMessage | undefined): string => { * more than once spoke before each search, and that text is part of the turn: * dropping it hides the model's reasoning from the user and leaves the * client's transcript out of step with what the model actually said. + * + * The same hops are also re-grouped into `turns`, because the folded strings + * cannot express where one hop ends and the next begins. */ const withIntermediateTurns = ({ + executions, payload, reasonings, texts, }: { + executions: ServerToolExecution[][]; payload: ChatCompletionPayload; reasonings: string[]; texts: string[]; @@ -496,13 +539,21 @@ const withIntermediateTurns = ({ const extraReasoning = reasonings.filter(Boolean).join('\n\n'); const [first, ...rest] = payload.choices ?? []; - if (!first || (!extraText && !extraReasoning)) { + if (!first) { return payload; } const message = first.message ?? {}; const existingText = typeof message.content === 'string' ? message.content : ''; + + // Nothing from the earlier hops and nothing to fold in — but the hops may + // still have run tools, which is exactly the case a caller consuming `turns` + // needs: a hop that called a tool without speaking first is still a hop. + if (!extraText && !extraReasoning && !executions.length) { + return payload; + } + const content = [extraText, existingText].filter(Boolean).join('\n\n'); const reasoning = [extraReasoning, readReasoning(message)] .filter(Boolean) @@ -510,6 +561,16 @@ const withIntermediateTurns = ({ return { ...payload, + // Per-hop grouping for renderers that can express it. The joined strings + // above stay as the OpenAI-shaped view; a client that builds Anthropic + // content blocks needs to know where one hop's reasoning ends and the next + // begins, which a joined string has already lost. The closing hop is the + // model's final answer, so it carries no further calls. + turns: buildIntermediateTurns({ + executions, + reasonings: [...reasonings, readReasoning(message)], + texts: [...texts, existingText], + }), choices: [ { ...first, @@ -557,6 +618,20 @@ export type ServerToolExecution = result: WebFetchResponse; }); +/** + * One server-tool hop as the model produced it. + * + * `reasoning` and `text` are what the model wrote before the calls in + * `executions`; both are empty when it called tools without speaking first. The + * last hop of a turn usually has no executions, because the model answered + * instead of reaching for another tool. + */ +export interface ServerToolTurn { + executions: ServerToolExecution[]; + reasoning: string; + text: string; +} + export interface ServerToolCallbacks { emitStreamEvents?: boolean; onCall?: (invocation: ServerToolInvocation) => void; @@ -1469,6 +1544,10 @@ export const executeWebSearchLoop = async ({ // turn has to carry its earlier steps forward explicitly. const intermediateTexts: string[] = []; const intermediateReasonings: string[] = []; + // The calls each hop made, parallel to the two arrays above. Block renderers + // need the calls grouped with the prose that produced them, not flattened + // into one list at the end. + const intermediateExecutions: ServerToolExecution[][] = []; const initialMode: ServerToolUpstreamMode = searchProvider && fetchProvider ? 'detect-both' @@ -1596,6 +1675,7 @@ export const executeWebSearchLoop = async ({ // current one is folded in by the helper itself. message: withIntermediateTurns({ payload, + executions: intermediateExecutions, reasonings: intermediateReasonings, texts: intermediateTexts, }).choices?.[0]?.message, @@ -1609,15 +1689,16 @@ export const executeWebSearchLoop = async ({ }; } - // This iteration is complete and the loop continues, so its text becomes - // part of what the final answer has to carry. - if (iterationText) { - intermediateTexts.push(iterationText); - } + // This iteration is complete and the loop continues, so its prose and the + // calls it made both become part of the turn the client sees. Keep the three + // arrays index-aligned: entry N is hop N, so a renderer can pair that hop's + // reasoning, text, and tool calls without guessing. A hop that called tools + // without speaking first still gets an entry — its prose sides stay empty. + const hop = intermediateTexts.length; - if (iterationReasoning) { - intermediateReasonings.push(iterationReasoning); - } + intermediateTexts[hop] = iterationText; + intermediateReasonings[hop] = iterationReasoning; + intermediateExecutions[hop] = results.map((result) => result.execution); messages.push(message as JsonRecord); messages.push( @@ -1666,6 +1747,7 @@ export const executeWebSearchLoop = async ({ { ...withIntermediateTurns({ payload, + executions: intermediateExecutions, reasonings: intermediateReasonings, texts: intermediateTexts, }), @@ -1683,6 +1765,7 @@ export const executeWebSearchLoop = async ({ { ...withIntermediateTurns({ payload, + executions: intermediateExecutions, reasonings: intermediateReasonings, texts: intermediateTexts, }), diff --git a/tests/server/web-search.test.ts b/tests/server/web-search.test.ts index d25f194..7bc0cd0 100644 --- a/tests/server/web-search.test.ts +++ b/tests/server/web-search.test.ts @@ -5615,6 +5615,170 @@ describe('proxy integration', () => { expect(text).toContain('data: [DONE]'); }); + it('interleaves thinking and text around each fetch in a non-streaming reply', async () => { + await updateSettings({ CODEBUDDY_WEB_FETCH_BACKEND: 'codebuddy' }); + let upstreamCalls = 0; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/webfetch')) { + // Backends report the URL they actually read, which follows redirects + // and so can differ from the one the model asked for. + return makeJsonResponse({ + content: 'Fetched body.', + url: 'https://page.test/a?redirected=1', + }); + } + + upstreamCalls += 1; + + // The model thinks, speaks, then fetches — twice over. Anthropic lays a + // turn out as thinking → text → tool_use → tool_result → thinking → + // text, so each hop's reasoning stays attached to the text it justifies. + if (upstreamCalls === 1) { + return makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + content: 'Looking it up.', + reasoning_content: 'I should check the page.', + role: 'assistant', + tool_calls: [ + { + id: 'call_first', + function: { + arguments: '{"url":"https://page.test/a"}', + name: 'web_fetch', + }, + }, + ], + }, + }, + ], + }); + } + + return makeJsonResponse({ + choices: [ + { + finish_reason: 'stop', + message: { + content: 'Here is what it said.', + reasoning_content: 'The page confirms it.', + }, + }, + ], + }); + }); + + const response = await handleMessagesRequest( + new NextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Read https://page.test/a' }], + tools: [ + { type: 'web_fetch_20260209', name: 'web_fetch', input_schema: {} }, + ], + }, + ); + + const payload = (await response.json()) as { + content: Array<{ + thinking?: string; + text?: string; + type: string; + content?: { url?: string }; + }>; + }; + const blocks = payload.content.map((block) => + block.type === 'thinking' + ? `thinking:${block.thinking}` + : block.type === 'text' + ? `text:${block.text}` + : block.type, + ); + + // Each hop keeps its own reasoning ahead of its own text, and the fetch + // sits between the two hops rather than ahead of both. + expect(blocks).toEqual([ + 'thinking:I should check the page.', + 'text:Looking it up.', + 'server_tool_use', + 'web_fetch_tool_result', + 'thinking:The page confirms it.', + 'text:Here is what it said.', + ]); + // The result carries the URL the backend read, not the one requested. + expect(payload.content[3]?.content?.url).toBe( + 'https://page.test/a?redirected=1', + ); + expect(upstreamCalls).toBe(2); + }); + + it('keeps the tool blocks first when a hop calls a tool without speaking', async () => { + await updateSettings({ CODEBUDDY_WEB_FETCH_BACKEND: 'codebuddy' }); + let upstreamCalls = 0; + + vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { + const url = String(input); + + if (url.includes('/agenttool/v1/webfetch')) { + return makeJsonResponse({ content: 'Fetched body.' }); + } + + upstreamCalls += 1; + + return upstreamCalls === 1 + ? makeJsonResponse({ + choices: [ + { + finish_reason: 'tool_calls', + message: { + role: 'assistant', + tool_calls: [ + { + id: 'call_fetch', + function: { + arguments: '{"url":"https://page.test/a"}', + name: 'web_fetch', + }, + }, + ], + }, + }, + ], + }) + : makeJsonResponse({ + choices: [{ finish_reason: 'stop', message: { content: 'Done.' } }], + }); + }); + + const response = await handleMessagesRequest( + new NextRequest('http://localhost/v1/messages', { method: 'POST' }), + { + max_tokens: 1024, + messages: [{ role: 'user', content: 'Read https://page.test/a' }], + tools: [ + { type: 'web_fetch_20260209', name: 'web_fetch', input_schema: {} }, + ], + }, + ); + + const payload = (await response.json()) as { + content: Array<{ text?: string; type: string }>; + }; + + // The model went straight to the tool, so there is no prose to put first: + // the fetch opens the turn and the answer closes it. + expect( + payload.content.map((block) => + block.type === 'text' ? `text:${block.text}` : block.type, + ), + ).toEqual(['server_tool_use', 'web_fetch_tool_result', 'text:Done.']); + }); + it('passes through untouched when no search tool is declared', async () => { process.env.SEARXNG_URL = 'https://searx.test'; resetWebSearchProviders();