Cover cache-aware effective cost with unit tests and fix Anthropic cache-read field

src/agentlab/llm/tracking.py

@@ -320,7 +320,9 @@ def get_effective_cost_from_antrophic_api(self, response) -> float:
         usage = getattr(response, "usage", {})
         new_input_tokens = getattr(usage, "input_tokens", 0)  # new input tokens
         output_tokens = getattr(usage, "output_tokens", 0)
-        cache_read_tokens = getattr(usage, "cache_input_tokens", 0)
+        cache_read_tokens = getattr(usage, "cache_read_input_tokens", None)
+        if cache_read_tokens is None:
+            cache_read_tokens = getattr(usage, "cache_input_tokens", 0)
         cache_write_tokens = getattr(usage, "cache_creation_input_tokens", 0)
 
         cache_read_cost = self.input_cost * ANTHROPIC_CACHE_PRICING_FACTOR["cache_read_tokens"]

tests/llm/test_response_api.py

@@ -168,7 +168,7 @@ def create_mock_anthropic_response(
     response.usage = MagicMock()
     response.usage.input_tokens = input_tokens
     response.usage.output_tokens = output_tokens
-    response.usage.cache_input_tokens = 0
+    response.usage.cache_read_input_tokens = 0
     response.usage.cache_creation_input_tokens = 0
     return response
 

tests/llm/test_tracking.py

@@ -175,3 +175,154 @@ def test_openrouter_chat_model():
         answer = chat_model(messages)
     assert "5" in answer.get("content")
     assert tracker.stats["cost"] > 0
+
+
+class _FakeUsage:
+    """Usage object that supports both attribute access and dict()."""
+
+    def __init__(self, **fields):
+        self.__dict__.update(fields)
+
+    def __iter__(self):
+        return iter(self.__dict__.items())
+
+
+class _FakeResponse:
+    def __init__(self, usage=None):
+        if usage is not None:
+            self.usage = usage
+
+
+class _CostModel(tracking.TrackAPIPricingMixin):
+    """Minimal model exposing the effective cost logic without hitting any provider."""
+
+    def __init__(self, pricing_api, input_cost, output_cost, response=None):
+        self._pricing_api = pricing_api
+        self.input_cost = input_cost
+        self.output_cost = output_cost
+        self._response = response
+        self.reset_stats()
+
+    def _call_api(self, *args, **kwargs):
+        return self._response
+
+    def _parse_response(self, response):
+        return response
+
+
+def test_effective_cost_anthropic_with_cache_tokens():
+    model = _CostModel("anthropic", input_cost=1e-5, output_cost=2e-5)
+    usage = _FakeUsage(
+        input_tokens=100,
+        output_tokens=50,
+        cache_read_input_tokens=1000,
+        cache_creation_input_tokens=200,
+    )
+
+    cost = model.get_effective_cost(_FakeResponse(usage))
+
+    expected = (
+        100 * 1e-5
+        + 50 * 2e-5
+        + 1000 * 1e-5 * tracking.ANTHROPIC_CACHE_PRICING_FACTOR["cache_read_tokens"]
+        + 200 * 1e-5 * tracking.ANTHROPIC_CACHE_PRICING_FACTOR["cache_write_tokens"]
+    )
+    assert cost == pytest.approx(expected)
+    # cache tokens must contribute, otherwise cached runs would be reported as cheaper than they are
+    assert cost > 100 * 1e-5 + 50 * 2e-5
+
+
+def test_effective_cost_anthropic_without_cache_tokens():
+    model = _CostModel("anthropic", input_cost=1e-5, output_cost=2e-5)
+    usage = _FakeUsage(input_tokens=100, output_tokens=50)
+
+    cost = model.get_effective_cost(_FakeResponse(usage))
+
+    assert cost == pytest.approx(100 * 1e-5 + 50 * 2e-5)
+
+
+def test_effective_cost_openai_chat_completion_with_cached_tokens():
+    model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5)
+    usage = _FakeUsage(
+        prompt_tokens=1000,
+        completion_tokens=50,
+        prompt_tokens_details=_FakeUsage(cached_tokens=800),
+    )
+
+    cost = model.get_effective_cost(_FakeResponse(usage))
+
+    expected = (
+        200 * 1e-5
+        + 800 * 1e-5 * tracking.OPENAI_CACHE_PRICING_FACTOR["cache_read_tokens"]
+        + 50 * 2e-5
+    )
+    assert cost == pytest.approx(expected)
+    # cached input tokens are cheaper than new ones
+    assert cost < 1000 * 1e-5 + 50 * 2e-5
+
+
+def test_effective_cost_openai_chat_completion_without_details():
+    model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5)
+    usage = _FakeUsage(prompt_tokens=1000, completion_tokens=50, prompt_tokens_details=None)
+
+    cost = model.get_effective_cost(_FakeResponse(usage))
+
+    assert cost == pytest.approx(1000 * 1e-5 + 50 * 2e-5)
+
+
+def test_effective_cost_openai_response_api_with_cached_tokens():
+    model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5)
+    usage = _FakeUsage(
+        input_tokens=1000,
+        output_tokens=50,
+        input_tokens_details=_FakeUsage(cached_tokens=400),
+    )
+
+    cost = model.get_effective_cost(_FakeResponse(usage))
+
+    expected = (
+        600 * 1e-5
+        + 400 * 1e-5 * tracking.OPENAI_CACHE_PRICING_FACTOR["cache_read_tokens"]
+        + 50 * 2e-5
+    )
+    assert cost == pytest.approx(expected)
+
+
+def test_effective_cost_openai_without_usage():
+    model = _CostModel("openai", input_cost=1e-5, output_cost=2e-5)
+
+    assert model.get_effective_cost(_FakeResponse()) == 0.0
+
+
+def test_effective_cost_unsupported_provider():
+    model = _CostModel("some-unknown-provider", input_cost=1e-5, output_cost=2e-5)
+    usage = _FakeUsage(input_tokens=100, output_tokens=50)
+
+    assert model.get_effective_cost(_FakeResponse(usage)) == 0.0
+
+
+def test_effective_cost_litellm_delegates_to_completion_cost(monkeypatch):
+    model = _CostModel("litellm", input_cost=1e-5, output_cost=2e-5)
+    response = _FakeResponse(_FakeUsage(input_tokens=100, output_tokens=50))
+    monkeypatch.setattr(tracking, "completion_cost", lambda resp: 0.42)
+
+    assert model.get_effective_cost(response) == 0.42
+
+
+def test_call_accumulates_effective_cost_in_stats():
+    usage = _FakeUsage(
+        input_tokens=100,
+        output_tokens=50,
+        cache_read_input_tokens=1000,
+        cache_creation_input_tokens=200,
+    )
+    model = _CostModel(
+        "anthropic", input_cost=1e-5, output_cost=2e-5, response=_FakeResponse(usage)
+    )
+    expected = model.get_effective_cost(_FakeResponse(usage))
+
+    model()
+    model()
+
+    assert model.stats.stats_dict["effective_cost"] == pytest.approx(2 * expected)
+    assert model.stats.stats_dict["n_api_calls"] == 2