From cdc39432b77d0a301b92e14b50fdca78728da8f4 Mon Sep 17 00:00:00 2001 From: Duane Millar Barlow Date: Thu, 12 Nov 2026 17:29:57 -0500 Subject: [PATCH] Cache the repeated prompt prefix on Claude chat requests One question is several model requests, each re-sending the tool definitions, the instructions, and every earlier tool result. Mark all three as cacheable through OpenRouter so later requests read them at the cached rate. Other providers ignore the markers. The turn's usage now reports cache read and write tokens. --- backend/chat/agent.py | 10 ++++ backend/tests/test_chat.py | 90 ++++++++++++++++++++++++++++++++++ docs/architecture/data-chat.md | 7 +++ 3 files changed, 107 insertions(+) diff --git a/backend/chat/agent.py b/backend/chat/agent.py index 70690790..4a29110a 100644 --- a/backend/chat/agent.py +++ b/backend/chat/agent.py @@ -245,6 +245,13 @@ def build_agent(settings: ChatSettings, service: DataToolService) -> Agent: # Ask OpenRouter to report each response's cost (see turn_cost). model_settings["openrouter_usage"] = {"include": True} model_settings["openrouter_provider"] = PROVIDER_ROUTING + # One question is several model requests, each re-sending the tool + # definitions, the instructions, and every earlier tool result. Mark + # all three as cacheable so later requests read that prefix at the + # cached rate. Providers without explicit cache control ignore these. + model_settings["openrouter_cache_tool_definitions"] = True + model_settings["openrouter_cache_instructions"] = True + model_settings["openrouter_cache_messages"] = True tools = [ data_tool(service, name, settings.preview_rows) for name in AGENT_DATA_TOOLS ] @@ -354,6 +361,9 @@ async def stream_answer( "tool_calls": usage.tool_calls, "input_tokens": usage.input_tokens, "output_tokens": usage.output_tokens, + # Part of input_tokens; shows whether caching works. + "cache_read_tokens": usage.cache_read_tokens, + "cache_write_tokens": usage.cache_write_tokens, "cost_usd": turn_cost(event.result.new_messages()), }, } diff --git a/backend/tests/test_chat.py b/backend/tests/test_chat.py index 19765daf..a0bc6792 100644 --- a/backend/tests/test_chat.py +++ b/backend/tests/test_chat.py @@ -4,6 +4,7 @@ import json import duckdb +import httpx2 import pytest from fastapi import FastAPI from fastapi.testclient import TestClient @@ -11,6 +12,8 @@ from pydantic_ai import ToolReturn from pydantic_ai.exceptions import ModelHTTPError from pydantic_ai.models.function import FunctionModel +from pydantic_ai.models.openrouter import OpenRouterModel +from pydantic_ai.providers.openrouter import OpenRouterProvider from chat import access from chat.agent import build_agent, chart_tool, preview, rank_tool, stream_answer @@ -424,3 +427,90 @@ def test_openrouter_models_report_cost_and_skip_data_collecting_providers(wareho agent = build_agent(settings, DataToolService(warehouse)) assert agent.model_settings["openrouter_provider"] == {"data_collection": "deny"} assert agent.model_settings["openrouter_usage"] == {"include": True} + + +def openrouter_requests(warehouse, slug): + """Play one tool-using turn against a fake OpenRouter; return the request + bodies it received and the turn's events.""" + bodies = [] + + def respond(request): + body = json.loads(request.content) + bodies.append(body) + delta = ( + { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "call_1", + "type": "function", + "function": {"name": "list_datasets", "arguments": "{}"}, + } + ], + } + if len(bodies) == 1 + else {"role": "assistant", "content": "Done."} + ) + chunk = { + "id": "x", + "object": "chat.completion.chunk", + "created": 1, + "model": body["model"], + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": "tool_calls" if len(bodies) == 1 else "stop", + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 1, + "total_tokens": 11, + "prompt_tokens_details": {"cached_tokens": 7, "cache_write_tokens": 2}, + }, + } + return httpx2.Response( + 200, + content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n", + headers={"content-type": "text/event-stream"}, + ) + + settings = ChatSettings(model=slug, warehouse_path=warehouse) + agent = build_agent(settings, DataToolService(warehouse)) + + async def run(): + client = httpx2.AsyncClient(transport=httpx2.MockTransport(respond)) + model = OpenRouterModel( + slug, provider=OpenRouterProvider(api_key="test", http_client=client) + ) + with agent.override(model=model): + return [e async for e in stream_answer(agent, settings, "hi", [])] + + return bodies, asyncio.run(run()) + + +def cache_marked(part) -> bool: + return "cache_control" in json.dumps(part) + + +def test_claude_requests_mark_the_repeated_prefix_as_cacheable(warehouse): + bodies, events = openrouter_requests(warehouse, "anthropic/claude-haiku-5.5") + assert len(bodies) == 2 + for body in bodies: + # The last tool definition, the instructions, and the newest message: + # everything before each marker is read from cache on the next request. + assert [cache_marked(tool) for tool in body["tools"]][-1] + assert cache_marked(body["messages"][0]) + assert cache_marked(body["messages"][-1]) + assert bodies[1]["messages"][-1]["role"] == "tool" + usage = events[-1]["usage"] + assert usage["cache_read_tokens"] == 14 + assert usage["cache_write_tokens"] == 4 + + +def test_other_models_get_no_cache_markers(warehouse): + bodies, events = openrouter_requests(warehouse, "openai/gpt-6.1-sol") + assert events[-1]["type"] == "done" + assert not any(cache_marked(body) for body in bodies) diff --git a/docs/architecture/data-chat.md b/docs/architecture/data-chat.md index 680a77f7..edc0484d 100644 --- a/docs/architecture/data-chat.md +++ b/docs/architecture/data-chat.md @@ -223,6 +223,13 @@ against live warehouse data, with `data_collection: "deny"` routing. hosts, many running reduced-precision (fp4) builds, so quality can vary by host. +These runs predate prompt caching. Claude requests now mark the tool +definitions, the instructions, and the newest message as cacheable +(`build_agent` in `chat/agent.py`), so each later request of a turn reads the +repeated prefix at the cached rate; the turn's `usage` reports +`cache_read_tokens` and `cache_write_tokens`. Other providers ignore the +markers. Re-run the evals to measure the new cost. + Costs are per eval chat. Real questions may be longer or chattier, so treat the last column as an upper bound on volume until we log real usage. The earlier planning estimate (about $0.08 per question for Sol) was roughly double Sol's