From ebdde9a4afb02cd8c9b0012c67e870f97e5bb969 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:46:12 +0000 Subject: [PATCH 1/2] bug: fix Anthropic cache-read usage field and add effective-cost tests --- src/agentlab/llm/tracking.py | 4 +- tests/llm/test_tracking.py | 151 +++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) diff --git a/src/agentlab/llm/tracking.py b/src/agentlab/llm/tracking.py index bc8e31715..d57c88957 100644 --- a/src/agentlab/llm/tracking.py +++ b/src/agentlab/llm/tracking.py @@ -320,7 +320,9 @@ def get_effective_cost_from_antrophic_api(self, response) -> float: usage = getattr(response, "usage", {}) new_input_tokens = getattr(usage, "input_tokens", 0) # new input tokens output_tokens = getattr(usage, "output_tokens", 0) - cache_read_tokens = getattr(usage, "cache_input_tokens", 0) + cache_read_tokens = getattr(usage, "cache_read_input_tokens", None) + if cache_read_tokens is None: + cache_read_tokens = getattr(usage, "cache_input_tokens", 0) cache_write_tokens = getattr(usage, "cache_creation_input_tokens", 0) cache_read_cost = self.input_cost * ANTHROPIC_CACHE_PRICING_FACTOR["cache_read_tokens"] diff --git a/tests/llm/test_tracking.py b/tests/llm/test_tracking.py index 0809c54a6..73d91d146 100644 --- a/tests/llm/test_tracking.py +++ b/tests/llm/test_tracking.py @@ -175,3 +175,154 @@ def test_openrouter_chat_model(): answer = chat_model(messages) assert "5" in answer.get("content") assert tracker.stats["cost"] > 0 + + +class _FakeUsage: + """Usage object that supports both attribute access and dict().""" + + def __init__(self, **fields): + self.__dict__.update(fields) + + def __iter__(self): + return iter(self.__dict__.items()) + + +class _FakeResponse: + def __init__(self, usage=None): + if usage is not None: + self.usage = usage + + +class _CostModel(tracking.TrackAPIPricingMixin): + """Minimal model exposing the effective cost logic without hitting any provider.""" + + def __init__(self, pricing_api, input_cost, output_cost, response=None): + self._pricing_api = pricing_api + self.input_cost = input_cost + self.output_cost = output_cost + self._response = response + self.reset_stats() + + def _call_api(self, *args, **kwargs): + return self._response + + def _parse_response(self, response): + return response + + +def test_effective_cost_anthropic_with_cache_tokens(): + model = _CostModel("anthropic", input_cost=1e-5, output_cost=2e-5) + usage = _FakeUsage( + input_tokens=100, + output_tokens=50, + cache_read_input_tokens=1000, + cache_creation_input_tokens=200, + ) + + cost = model.get_effective_cost(_FakeResponse(usage)) + + expected = ( + 100 * 1e-5 + + 50 * 2e-5 + + 1000 * 1e-5 * tracking.ANTHROPIC_CACHE_PRICING_FACTOR["cache_read_tokens"] + + 200 * 1e-5 * tracking.ANTHROPIC_CACHE_PRICING_FACTOR["cache_write_tokens"] + ) + assert cost == pytest.approx(expected) + # cache tokens must contribute, otherwise cached runs would be reported as cheaper than they are + assert cost > 100 * 1e-5 + 50 * 2e-5 + + +def test_effective_cost_anthropic_without_cache_tokens(): + model = _CostModel("anthropic", input_cost=1e-5, output_cost=2e-5) + usage = _FakeUsage(input_tokens=100, output_tokens=50) + + cost = model.get_effective_cost(_FakeResponse(usage)) + + assert cost == pytest.approx(100 * 1e-5 + 50 * 2e-5) + + +def test_effective_cost_openai_chat_completion_with_cached_tokens(): + model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5) + usage = _FakeUsage( + prompt_tokens=1000, + completion_tokens=50, + prompt_tokens_details=_FakeUsage(cached_tokens=800), + ) + + cost = model.get_effective_cost(_FakeResponse(usage)) + + expected = ( + 200 * 1e-5 + + 800 * 1e-5 * tracking.OPENAI_CACHE_PRICING_FACTOR["cache_read_tokens"] + + 50 * 2e-5 + ) + assert cost == pytest.approx(expected) + # cached input tokens are cheaper than new ones + assert cost < 1000 * 1e-5 + 50 * 2e-5 + + +def test_effective_cost_openai_chat_completion_without_details(): + model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5) + usage = _FakeUsage(prompt_tokens=1000, completion_tokens=50, prompt_tokens_details=None) + + cost = model.get_effective_cost(_FakeResponse(usage)) + + assert cost == pytest.approx(1000 * 1e-5 + 50 * 2e-5) + + +def test_effective_cost_openai_response_api_with_cached_tokens(): + model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5) + usage = _FakeUsage( + input_tokens=1000, + output_tokens=50, + input_tokens_details=_FakeUsage(cached_tokens=400), + ) + + cost = model.get_effective_cost(_FakeResponse(usage)) + + expected = ( + 600 * 1e-5 + + 400 * 1e-5 * tracking.OPENAI_CACHE_PRICING_FACTOR["cache_read_tokens"] + + 50 * 2e-5 + ) + assert cost == pytest.approx(expected) + + +def test_effective_cost_openai_without_usage(): + model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5) + + assert model.get_effective_cost(_FakeResponse()) == 0.0 + + +def test_effective_cost_unsupported_provider(): + model = _CostModel("some-unknown-provider", input_cost=1e-5, output_cost=2e-5) + usage = _FakeUsage(input_tokens=100, output_tokens=50) + + assert model.get_effective_cost(_FakeResponse(usage)) == 0.0 + + +def test_effective_cost_litellm_delegates_to_completion_cost(monkeypatch): + model = _CostModel("litellm", input_cost=1e-5, output_cost=2e-5) + response = _FakeResponse(_FakeUsage(input_tokens=100, output_tokens=50)) + monkeypatch.setattr(tracking, "completion_cost", lambda resp: 0.42) + + assert model.get_effective_cost(response) == 0.42 + + +def test_call_accumulates_effective_cost_in_stats(): + usage = _FakeUsage( + input_tokens=100, + output_tokens=50, + cache_read_input_tokens=1000, + cache_creation_input_tokens=200, + ) + model = _CostModel( + "anthropic", input_cost=1e-5, output_cost=2e-5, response=_FakeResponse(usage) + ) + expected = model.get_effective_cost(_FakeResponse(usage)) + + model() + model() + + assert model.stats.stats_dict["effective_cost"] == pytest.approx(2 * expected) + assert model.stats.stats_dict["n_api_calls"] == 2 From 3656be02e6976c80f01b4f2ee95116b4dcbdc67e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 20:49:59 +0000 Subject: [PATCH 2/2] bug: use anthropic cache_read_input_tokens in mock usage --- tests/llm/test_response_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/llm/test_response_api.py b/tests/llm/test_response_api.py index 6174f64de..d9368f36b 100644 --- a/tests/llm/test_response_api.py +++ b/tests/llm/test_response_api.py @@ -168,7 +168,7 @@ def create_mock_anthropic_response( response.usage = MagicMock() response.usage.input_tokens = input_tokens response.usage.output_tokens = output_tokens - response.usage.cache_input_tokens = 0 + response.usage.cache_read_input_tokens = 0 response.usage.cache_creation_input_tokens = 0 return response