Cache the repeated prompt prefix on Claude chat requests

backend/chat/agent.py

@@ -245,6 +245,13 @@ def build_agent(settings: ChatSettings, service: DataToolService) -> Agent:
         # Ask OpenRouter to report each response's cost (see turn_cost).
         model_settings["openrouter_usage"] = {"include": True}
         model_settings["openrouter_provider"] = PROVIDER_ROUTING
+        # One question is several model requests, each re-sending the tool
+        # definitions, the instructions, and every earlier tool result. Mark
+        # all three as cacheable so later requests read that prefix at the
+        # cached rate. Providers without explicit cache control ignore these.
+        model_settings["openrouter_cache_tool_definitions"] = True
+        model_settings["openrouter_cache_instructions"] = True
+        model_settings["openrouter_cache_messages"] = True
     tools = [
         data_tool(service, name, settings.preview_rows) for name in AGENT_DATA_TOOLS
     ]
@@ -354,6 +361,9 @@ async def stream_answer(
                             "tool_calls": usage.tool_calls,
                             "input_tokens": usage.input_tokens,
                             "output_tokens": usage.output_tokens,
+                            # Part of input_tokens; shows whether caching works.
+                            "cache_read_tokens": usage.cache_read_tokens,
+                            "cache_write_tokens": usage.cache_write_tokens,
                             "cost_usd": turn_cost(event.result.new_messages()),
                         },
                     }

backend/tests/test_chat.py

@@ -4,13 +4,16 @@
 import json
 
 import duckdb
+import httpx2
 import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 from pydantic import ValidationError
 from pydantic_ai import ToolReturn
 from pydantic_ai.exceptions import ModelHTTPError
 from pydantic_ai.models.function import FunctionModel
+from pydantic_ai.models.openrouter import OpenRouterModel
+from pydantic_ai.providers.openrouter import OpenRouterProvider
 
 from chat import access
 from chat.agent import build_agent, chart_tool, preview, rank_tool, stream_answer
@@ -424,3 +427,90 @@ def test_openrouter_models_report_cost_and_skip_data_collecting_providers(wareho
     agent = build_agent(settings, DataToolService(warehouse))
     assert agent.model_settings["openrouter_provider"] == {"data_collection": "deny"}
     assert agent.model_settings["openrouter_usage"] == {"include": True}
+
+
+def openrouter_requests(warehouse, slug):
+    """Play one tool-using turn against a fake OpenRouter; return the request
+    bodies it received and the turn's events."""
+    bodies = []
+
+    def respond(request):
+        body = json.loads(request.content)
+        bodies.append(body)
+        delta = (
+            {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                        "index": 0,
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {"name": "list_datasets", "arguments": "{}"},
+                    }
+                ],
+            }
+            if len(bodies) == 1
+            else {"role": "assistant", "content": "Done."}
+        )
+        chunk = {
+            "id": "x",
+            "object": "chat.completion.chunk",
+            "created": 1,
+            "model": body["model"],
+            "choices": [
+                {
+                    "index": 0,
+                    "delta": delta,
+                    "finish_reason": "tool_calls" if len(bodies) == 1 else "stop",
+                }
+            ],
+            "usage": {
+                "prompt_tokens": 10,
+                "completion_tokens": 1,
+                "total_tokens": 11,
+                "prompt_tokens_details": {"cached_tokens": 7, "cache_write_tokens": 2},
+            },
+        }
+        return httpx2.Response(
+            200,
+            content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n",
+            headers={"content-type": "text/event-stream"},
+        )
+
+    settings = ChatSettings(model=slug, warehouse_path=warehouse)
+    agent = build_agent(settings, DataToolService(warehouse))
+
+    async def run():
+        client = httpx2.AsyncClient(transport=httpx2.MockTransport(respond))
+        model = OpenRouterModel(
+            slug, provider=OpenRouterProvider(api_key="test", http_client=client)
+        )
+        with agent.override(model=model):
+            return [e async for e in stream_answer(agent, settings, "hi", [])]
+
+    return bodies, asyncio.run(run())
+
+
+def cache_marked(part) -> bool:
+    return "cache_control" in json.dumps(part)
+
+
+def test_claude_requests_mark_the_repeated_prefix_as_cacheable(warehouse):
+    bodies, events = openrouter_requests(warehouse, "anthropic/claude-haiku-5.5")
+    assert len(bodies) == 2
+    for body in bodies:
+        # The last tool definition, the instructions, and the newest message:
+        # everything before each marker is read from cache on the next request.
+        assert [cache_marked(tool) for tool in body["tools"]][-1]
+        assert cache_marked(body["messages"][0])
+        assert cache_marked(body["messages"][-1])
+    assert bodies[1]["messages"][-1]["role"] == "tool"
+    usage = events[-1]["usage"]
+    assert usage["cache_read_tokens"] == 14
+    assert usage["cache_write_tokens"] == 4
+
+
+def test_other_models_get_no_cache_markers(warehouse):
+    bodies, events = openrouter_requests(warehouse, "openai/gpt-6.1-sol")
+    assert events[-1]["type"] == "done"
+    assert not any(cache_marked(body) for body in bodies)

docs/architecture/data-chat.md

@@ -223,6 +223,13 @@ against live warehouse data, with `data_collection: "deny"` routing.
   hosts, many running reduced-precision (fp4) builds, so quality can vary by
   host.
 
+These runs predate prompt caching. Claude requests now mark the tool
+definitions, the instructions, and the newest message as cacheable
+(`build_agent` in `chat/agent.py`), so each later request of a turn reads the
+repeated prefix at the cached rate; the turn's `usage` reports
+`cache_read_tokens` and `cache_write_tokens`. Other providers ignore the
+markers. Re-run the evals to measure the new cost.
+
 Costs are per eval chat. Real questions may be longer or chattier, so treat
 the last column as an upper bound on volume until we log real usage. The earlier
 planning estimate (about $0.08 per question for Sol) was roughly double Sol's