diff --git a/tests/llm/test_response_api_unit.py b/tests/llm/test_response_api_unit.py
new file mode 100644
index 000000000..a383e219c
--- /dev/null
+++ b/tests/llm/test_response_api_unit.py
@@ -0,0 +1,899 @@
+"""Hermetic unit tests for agentlab.llm.response_api (no network, no API keys)."""
+
+import json
+import logging
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import numpy as np
+import pytest
+from anthropic.types import Usage as AnthropicUsage
+from openai.types.chat import ChatCompletion
+from openai.types.responses import ResponseUsage
+
+import agentlab.llm.response_api as ra
+import agentlab.llm.tracking as tracking
+from agentlab.llm.response_api import (
+ AgentlabAction,
+ AnthropicAPIMessageBuilder,
+ APIPayload,
+ BaseResponseModel,
+ ClaudeResponseModel,
+ ClaudeResponseModelArgs,
+ LLMOutput,
+ MessageBuilder,
+ OpenAIChatCompletionAPIMessageBuilder,
+ OpenAIChatCompletionModel,
+ OpenAIChatModelArgs,
+ OpenAIResponseAPIMessageBuilder,
+ OpenAIResponseModel,
+ OpenAIResponseModelArgs,
+ OpenRouterModelArgs,
+ ToolCall,
+ ToolCalls,
+ tool_call_to_python_code,
+)
+
+PRICES = {"prompt": 1e-3, "completion": 2e-3}
+
+
+@pytest.fixture(autouse=True)
+def _offline_pricing(monkeypatch):
+ """Keep model construction hermetic: no pricing lookups, no stray env keys."""
+ table = {name: PRICES for name in ("gpt-test", "claude-test", "router/test")}
+ monkeypatch.setattr(tracking, "get_pricing_openai", lambda: table)
+ monkeypatch.setattr(tracking, "get_pricing_anthropic", lambda: table)
+ previous = getattr(tracking.TRACKER, "instance", None)
+ tracking.TRACKER.instance = None
+ yield
+ tracking.TRACKER.instance = previous
+
+
+def _ns(**kwargs):
+ return SimpleNamespace(**kwargs)
+
+
+# --------------------------------------------------------------------------- ToolCall / ToolCalls
+
+
+class TestToolCall:
+ def test_response_setters_and_flag(self):
+ call = ToolCall(name="click", arguments={"bid": "a1"})
+ assert not call.is_response_set
+ assert call.response_text("ok") is call
+ assert call.tool_response == {"text": "ok"} and call.is_response_set
+ call.response_image("data:img")
+ assert call.tool_response == {"image": "data:img"}
+
+ def test_repr_hides_raw_call(self):
+ call = ToolCall(name="fill", arguments={"text": "x"}, raw_call=object())
+ assert repr(call) == "ToolCall(name=fill, arguments={'text': 'x'})"
+
+ def test_collection_protocol(self):
+ calls = ToolCalls()
+ assert not calls and len(calls) == 0 and calls.all_responses_set # vacuous truth
+ a, b = ToolCall(name="a"), ToolCall(name="b")
+ assert calls.add_tool_call(a).add_tool_call(b) is calls
+ assert bool(calls) and len(calls) == 2 and list(calls) == [a, b]
+ a.response_text("done")
+ assert not calls.all_responses_set
+ b.response_image("img")
+ assert calls.all_responses_set
+
+ def test_defaults_not_shared(self):
+ assert ToolCalls().tool_calls is not ToolCalls().tool_calls
+ assert ToolCall().arguments is not ToolCall().arguments
+ out = LLMOutput()
+ assert (out.raw_response, out.think, out.action, out.tool_calls) == (None, "", None, None)
+
+
+# --------------------------------------------------------------------------- MessageBuilder base
+
+
+class TestMessageBuilderBase:
+ @pytest.mark.parametrize(
+ "factory, role", [("system", "system"), ("user", "user"), ("assistant", "assistant")]
+ )
+ def test_role_factories(self, factory, role):
+ msg = getattr(MessageBuilder, factory)()
+ assert msg.role == role and msg.content == [] and msg.responded_tool_calls is None
+
+ def test_add_text_and_image_chain(self):
+ msg = MessageBuilder.user().add_text("hi").add_image("data:x")
+ assert msg.content == [{"text": "hi"}, {"image": "data:x"}]
+
+ def test_add_image_url_encodes_png(self):
+ msg = MessageBuilder.user().add_image_url(np.zeros((2, 2, 3), dtype=np.uint8))
+ assert msg.content[0]["image"].startswith("data:image/png;base64,")
+
+ def test_base_prepare_and_caching_not_implemented(self):
+ with pytest.raises(NotImplementedError):
+ MessageBuilder.user().prepare_message()
+ with pytest.raises(NotImplementedError):
+ MessageBuilder.user().mark_all_previous_msg_for_caching()
+
+ def test_add_responded_tool_calls_requires_all_responses(self):
+ calls = ToolCalls(tool_calls=[ToolCall(name="a")])
+ with pytest.raises(AssertionError, match="must have a response"):
+ MessageBuilder.add_responded_tool_calls(calls)
+ calls.tool_calls[0].response_text("ok")
+ msg = OpenAIChatCompletionAPIMessageBuilder.add_responded_tool_calls(calls)
+ assert isinstance(msg, OpenAIChatCompletionAPIMessageBuilder)
+ assert msg.role == "tool" and msg.responded_tool_calls is calls
+
+ def test_to_markdown_text_and_image(self):
+ md = MessageBuilder.assistant().add_text("hello").add_image("http://img").to_markdown()
+ assert md == "### Assistant\n\n```\nhello\n```\n\n"
+
+ def test_to_markdown_tool_calls(self):
+ calls = ToolCalls(
+ tool_calls=[
+ ToolCall(name="click", arguments={"bid": "a1"}).response_text("clicked"),
+ ToolCall(name="noop", arguments={}).response_image("shot.png"),
+ ]
+ )
+ md = MessageBuilder.add_responded_tool_calls(calls).to_markdown()
+ assert md.startswith("### Tool\n")
+ assert "**Tool Call 1**: click(bid='a1')" in md
+ assert "**Tool Response 1:**\n```\nclicked\n```" in md
+ assert "**Tool Call 2**: noop()" in md
+ assert "" in md
+
+ def test_to_markdown_tool_call_without_response(self):
+ msg = MessageBuilder("tool")
+ msg.responded_tool_calls = ToolCalls(tool_calls=[ToolCall(name="x", arguments=None)])
+ md = msg.to_markdown()
+ assert "**Tool Call 1**: x()" in md and "Tool Response" not in md
+
+
+# --------------------------------------------------------------------------- OpenAI Responses builder
+
+
+def _responded(raw_calls, *calls):
+ return MessageBuilder.add_responded_tool_calls(
+ ToolCalls(tool_calls=list(calls), raw_calls=raw_calls)
+ )
+
+
+class TestOpenAIResponseAPIMessageBuilder:
+ def test_system_uses_developer_role(self):
+ msg = OpenAIResponseAPIMessageBuilder.system().add_text("rules")
+ assert msg.prepare_message() == [
+ {"role": "developer", "content": [{"type": "input_text", "text": "rules"}]}
+ ]
+
+ def test_assistant_text_is_output_text(self):
+ msg = OpenAIResponseAPIMessageBuilder.assistant().add_text("a").add_image("img")
+ assert msg.prepare_message()[0]["content"] == [
+ {"type": "output_text", "text": "a"},
+ {"type": "input_image", "image_url": "img"},
+ ]
+
+ def test_unsupported_content(self):
+ msg = OpenAIResponseAPIMessageBuilder.user()
+ msg.content.append({"audio": "x"})
+ with pytest.raises(ValueError, match="Unsupported content type"):
+ msg.prepare_message()
+
+ def test_tool_role_without_calls_raises(self):
+ with pytest.raises(ValueError, match="No tool calls"):
+ OpenAIResponseAPIMessageBuilder("tool").prepare_message()
+
+ def test_function_and_computer_call_outputs(self):
+ prior = [_ns(type="reasoning"), _ns(type="function_call")]
+ fn = ToolCall(name="f", raw_call=_ns(type="function_call", call_id="c1")).response_text("r")
+ cu = ToolCall(name="cu", raw_call=_ns(type="computer_call", call_id="c2")).response_image(
+ "i"
+ )
+ other = ToolCall(name="o", raw_call=_ns(type="custom", call_id="c3")).response_text("z")
+ msg = OpenAIResponseAPIMessageBuilder.add_responded_tool_calls(
+ ToolCalls(tool_calls=[fn, cu, other], raw_calls=_ns(output=prior))
+ )
+ out = msg.prepare_message()
+ assert out[:2] == prior
+ assert out[2:] == [
+ {"type": "function_call_output", "call_id": "c1", "output": "r"},
+ {
+ "type": "computer_call_output",
+ "call_id": "c2",
+ "output": {"type": "input_image", "image_url": "i"},
+ },
+ ]
+
+ @pytest.mark.parametrize(
+ "call_type, response, match",
+ [
+ ("function_call", {"image": "i"}, "Image output"),
+ ("computer_call", {"text": "t"}, "Text output"),
+ ],
+ )
+ def test_wrong_response_kind_asserts(self, call_type, response, match):
+ call = ToolCall(name="f", raw_call=_ns(type=call_type, call_id="c"), tool_response=response)
+ msg = OpenAIResponseAPIMessageBuilder.add_responded_tool_calls(
+ ToolCalls(tool_calls=[call], raw_calls=_ns(output=[]))
+ )
+ with pytest.raises(AssertionError, match=match):
+ msg.prepare_message()
+
+ def test_mark_caching_is_noop(self):
+ msg = OpenAIResponseAPIMessageBuilder.user().add_text("x")
+ msg.mark_all_previous_msg_for_caching()
+ assert not hasattr(msg, "_cache_breakpoint")
+
+
+# --------------------------------------------------------------------------- Anthropic builder
+
+
+class TestAnthropicAPIMessageBuilder:
+ def test_assistant_text_is_stripped(self):
+ msg = AnthropicAPIMessageBuilder.assistant().add_text(" answer \n").add_image("raw")
+ out = msg.prepare_message()
+ assert out[0]["content"][0] == {"type": "text", "text": "answer"}
+ assert msg.content[0]["text"] == " answer \n" # source content untouched
+
+ def test_user_text_not_stripped(self):
+ msg = AnthropicAPIMessageBuilder.user().add_text(" keep ")
+ assert msg.prepare_message() == [
+ {"role": "user", "content": [{"type": "text", "text": " keep "}]}
+ ]
+
+ @pytest.mark.parametrize(
+ "img, data", [("data:image/png;base64,QUJD", "QUJD"), ("QUJD", "QUJD")]
+ )
+ def test_image_prefix_removed(self, img, data):
+ assert AnthropicAPIMessageBuilder.user().transform_content({"image": img}) == {
+ "type": "image",
+ "source": {"type": "base64", "media_type": "image/png", "data": data},
+ }
+
+ def test_unsupported_content(self):
+ with pytest.raises(ValueError, match="Unsupported content type"):
+ AnthropicAPIMessageBuilder.user().transform_content({"video": "v"})
+
+ def test_tool_role_without_calls_raises(self):
+ with pytest.raises(ValueError, match="No tool calls"):
+ AnthropicAPIMessageBuilder("tool").prepare_message()
+
+ def test_tool_results(self):
+ blocks = [_ns(type="text"), _ns(type="tool_use")]
+ calls = ToolCalls(
+ tool_calls=[
+ ToolCall(name="a", raw_call=_ns(id="tu_1")).response_text("r1"),
+ ToolCall(name="b", raw_call=_ns(id="tu_2")).response_text("r2"),
+ ],
+ raw_calls=_ns(content=blocks),
+ )
+ out = AnthropicAPIMessageBuilder.add_responded_tool_calls(calls).prepare_message()
+ assert out == [
+ {"role": "assistant", "content": blocks},
+ {
+ "role": "user",
+ "content": [
+ {"type": "tool_result", "tool_use_id": "tu_1", "content": "r1"},
+ {"type": "tool_result", "tool_use_id": "tu_2", "content": "r2"},
+ ],
+ },
+ ]
+
+ def test_tool_image_response_asserts(self):
+ calls = ToolCalls(
+ tool_calls=[ToolCall(name="a", raw_call=_ns(id="t")).response_image("i")],
+ raw_calls=_ns(content=[]),
+ )
+ with pytest.raises(AssertionError, match="Image output"):
+ AnthropicAPIMessageBuilder.add_responded_tool_calls(calls).prepare_message()
+
+ def test_mark_caching_sets_breakpoint(self):
+ msg = AnthropicAPIMessageBuilder.user()
+ assert not getattr(msg, "_cache_breakpoint", False)
+ msg.mark_all_previous_msg_for_caching()
+ assert msg._cache_breakpoint is True
+
+
+# --------------------------------------------------------------------------- Chat Completion builder
+
+
+class TestOpenAIChatCompletionAPIMessageBuilder:
+ def test_unsupported_content(self):
+ with pytest.raises(ValueError, match="Unsupported content type"):
+ OpenAIChatCompletionAPIMessageBuilder.user().convert_content_to_expected_format(
+ {"x": 1}
+ )
+
+ def test_tool_role_without_calls_raises(self):
+ with pytest.raises(ValueError, match="No tool calls"):
+ OpenAIChatCompletionAPIMessageBuilder("tool").prepare_message()
+
+ def test_tool_outputs(self):
+ assistant_msg = {"role": "assistant", "tool_calls": ["..."]}
+ raw = _ns(choices=[_ns(message=assistant_msg)])
+ call = ToolCall(
+ name="click", raw_call={"id": "call_1", "function": {"name": "click"}}
+ ).response_text("done")
+ out = OpenAIChatCompletionAPIMessageBuilder.add_responded_tool_calls(
+ ToolCalls(tool_calls=[call], raw_calls=raw)
+ ).prepare_message()
+ assert out == [
+ assistant_msg,
+ {"name": "click", "role": "tool", "tool_call_id": "call_1", "content": "done"},
+ ]
+
+ def test_tool_image_response_asserts(self):
+ call = ToolCall(name="c", raw_call={"id": "1", "function": {"name": "c"}}).response_image(
+ "i"
+ )
+ msg = OpenAIChatCompletionAPIMessageBuilder.add_responded_tool_calls(
+ ToolCalls(tool_calls=[call], raw_calls=_ns(choices=[_ns(message={})]))
+ )
+ with pytest.raises(AssertionError, match="Image output"):
+ msg.prepare_message()
+
+ def test_mark_caching_is_noop(self):
+ msg = OpenAIChatCompletionAPIMessageBuilder.user()
+ assert msg.mark_all_previous_msg_for_caching() is None
+
+
+# --------------------------------------------------------------------------- APIPayload / helpers
+
+
+class TestAPIPayload:
+ def test_tool_choice_and_force_call_are_exclusive(self):
+ with pytest.raises(ValueError, match="mutually exclusive"):
+ APIPayload(tool_choice="auto", force_call_tool="click")
+
+ def test_reasoning_effort_logs_info(self, caplog):
+ with caplog.at_level(logging.INFO, logger=ra.logger.name):
+ payload = APIPayload(reasoning_effort="high")
+ assert payload.reasoning_effort == "high"
+ assert "reasoning_effort is used by LiteLLM API only" in caplog.text
+
+ def test_defaults(self):
+ p = APIPayload(force_call_tool="x")
+ assert (p.messages, p.tools, p.tool_choice, p.use_cache_breakpoints) == (
+ None,
+ None,
+ None,
+ False,
+ )
+
+
+@pytest.mark.parametrize(
+ "name, kwargs, expected",
+ [
+ ("noop", None, "noop()"),
+ ("noop", {}, "noop()"),
+ ("click", {"bid": "a1", "button": "left"}, "click(bid='a1', button='left')"),
+ ("scroll", {"dx": 0, "dy": -1.5, "flag": True}, "scroll(dx=0, dy=-1.5, flag=True)"),
+ ("fill", {"text": 'it\'s "q"\n'}, "fill(text='it\\'s \"q\"\\n')"),
+ ("multi", {"keys": ["a", "b"]}, "multi(keys=['a', 'b'])"),
+ ],
+)
+def test_tool_call_to_python_code(name, kwargs, expected):
+ assert tool_call_to_python_code(name, kwargs) == expected
+
+
+def test_agentlab_action_helpers():
+ call = ToolCall(name="go_back", arguments={})
+ assert AgentlabAction.convert_toolcall_to_agentlab_action_format(call) == "go_back()"
+ assert (
+ AgentlabAction.convert_multiactions_to_agentlab_action_format(["a()", "b()"]) == "a()\nb()"
+ )
+ assert AgentlabAction.convert_multiactions_to_agentlab_action_format([]) is None
+
+
+def test_base_response_model_call_chains_call_and_parse():
+ class Echo(BaseResponseModel):
+ def _call_api(self, payload):
+ return {"raw": payload}
+
+ def _parse_response(self, response):
+ return LLMOutput(raw_response=response, think="t")
+
+ model = Echo("m", api_key="k", temperature=0.3, max_tokens=7)
+ assert (model.model_name, model.api_key, model.temperature, model.max_tokens) == (
+ "m",
+ "k",
+ 0.3,
+ 7,
+ )
+ out = model("payload")
+ assert out.raw_response == {"raw": "payload"} and out.think == "t"
+
+
+# --------------------------------------------------------------------------- OpenAI Responses model
+
+
+def _responses_obj(outputs, inp=10, out=5):
+ usage = _ns(input_tokens=inp, output_tokens=out, input_tokens_details=_ns(cached_tokens=0))
+ return _ns(output=outputs, usage=usage)
+
+
+class TestOpenAIResponseModel:
+ def _model(self, **kw):
+ return OpenAIResponseModel(model_name="gpt-test", api_key="sk-test", **kw)
+
+ def test_init(self):
+ model = self._model(base_url="http://local/v1", temperature=0.2)
+ assert model.client.base_url.host == "local"
+ assert model.client.api_key == "sk-test"
+ assert (model.input_cost, model.output_cost) == (1e-3, 2e-3)
+ assert model._pricing_api == "openai" and model.max_tokens == 100
+
+ @pytest.mark.parametrize(
+ "payload_kw, expected_choice",
+ [
+ ({}, None),
+ ({"tool_choice": "auto"}, "auto"),
+ ({"tool_choice": "any"}, "required"),
+ ({"tool_choice": "required"}, "required"),
+ ({"force_call_tool": "click"}, {"type": "function", "name": "click"}),
+ ],
+ )
+ def test_call_api_params(self, payload_kw, expected_choice):
+ model = self._model(temperature=0.5, max_tokens=20)
+ msgs = [
+ OpenAIResponseAPIMessageBuilder.system().add_text("sys"),
+ OpenAIResponseAPIMessageBuilder.user().add_text("hi"),
+ ]
+ tools = [{"type": "function", "name": "click"}]
+ with patch.object(ra, "call_openai_api_with_retries", return_value="RAW") as call:
+ assert model._call_api(APIPayload(messages=msgs, tools=tools, **payload_kw)) == "RAW"
+ fn, params = call.call_args.args
+ assert fn == model.client.responses.create
+ assert params["model"] == "gpt-test"
+ assert params["input"] == [
+ {"role": "developer", "content": [{"type": "input_text", "text": "sys"}]},
+ {"role": "user", "content": [{"type": "input_text", "text": "hi"}]},
+ ]
+ assert params["temperature"] == 0.5 and params["max_output_tokens"] == 20
+ assert params["tools"] == tools
+ assert params.get("tool_choice") == expected_choice
+
+ def test_call_api_omits_unset_params(self):
+ model = self._model(max_tokens=None)
+ with patch.object(ra, "call_openai_api_with_retries", return_value="RAW") as call:
+ model._call_api(APIPayload(messages=[]))
+ assert call.call_args.args[1] == {"model": "gpt-test", "input": []}
+
+ def test_parse_function_calls_and_thinking(self):
+ fc1 = _ns(type="function_call", name="click", arguments=json.dumps({"bid": "a1"}))
+ fc2 = _ns(type="function_call", name="noop", arguments="{}")
+ response = _responses_obj(
+ [
+ _ns(type="reasoning", summary=[_ns(text="plan")]),
+ _ns(type="reasoning", summary=[]),
+ _ns(type="message", content=[_ns(text="hello")]),
+ _ns(type="message", content=[]),
+ _ns(type="other", output_text="extra"),
+ _ns(type="other", output_text=""),
+ fc1,
+ fc2,
+ ]
+ )
+ out = self._model()._parse_response(response)
+ assert out.think == "plan\nhello\nextra\n"
+ assert out.action == "click(bid='a1')\nnoop()"
+ assert [c.name for c in out.tool_calls] == ["click", "noop"]
+ assert out.tool_calls.tool_calls[0].raw_call is fc1
+ assert out.tool_calls.raw_calls is response and out.raw_response is response
+
+ def test_parse_single_call_and_no_call(self):
+ model = self._model()
+ one = model._parse_response(
+ _responses_obj([_ns(type="function_call", name="go_back", arguments="{}")])
+ )
+ assert one.action == "go_back()"
+ none = model._parse_response(_responses_obj([_ns(type="message", content=[_ns(text="x")])]))
+ assert none.action is None and len(none.tool_calls) == 0
+
+ def test_computer_call_requires_subclass(self):
+ response = _responses_obj([_ns(type="computer_call", action=_ns(type="click"))])
+ with pytest.raises(NotImplementedError):
+ self._model()._parse_response(response)
+
+ class CUA(OpenAIResponseModel):
+ def cua_action_to_env_tool_name_and_args(self, action):
+ return "mouse_click", {"kind": action.type}
+
+ model = CUA(model_name="gpt-test", api_key="sk-test")
+ assert model._parse_response(response).action == "mouse_click(kind='click')"
+
+ def test_text_action_space_returns_none(self):
+ model = self._model()
+ model.action_space_as_tools = False
+ response = _responses_obj([_ns(type="function_call", name="a", arguments="{}")])
+ out = model._parse_response(response)
+ assert out.action is None and len(out.tool_calls) == 1
+
+ def test_full_call_tracks_cost(self):
+ model = self._model()
+ response = _responses_obj([_ns(type="function_call", name="a", arguments="{}")], 100, 10)
+ response.usage = ResponseUsage(
+ input_tokens=100,
+ output_tokens=10,
+ total_tokens=110,
+ input_tokens_details={"cached_tokens": 0},
+ output_tokens_details={"reasoning_tokens": 0},
+ )
+ with patch.object(ra, "call_openai_api_with_retries", return_value=response):
+ with tracking.set_tracker() as tracker:
+ out = model(
+ APIPayload(messages=[OpenAIResponseAPIMessageBuilder.user().add_text("q")])
+ )
+ assert out.action == "a()"
+ assert tracker.stats == {
+ "input_tokens": 100,
+ "output_tokens": 10,
+ "cost": pytest.approx(0.12),
+ }
+ assert model.stats.stats_dict["n_api_calls"] == 1
+
+
+# --------------------------------------------------------------------------- Chat Completion model
+
+
+def _chat_completion(message, prompt=10, completion=5):
+ return ChatCompletion.model_validate(
+ {
+ "id": "c",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "gpt-test",
+ "choices": [{"index": 0, "finish_reason": "stop", "message": message}],
+ "usage": {
+ "prompt_tokens": prompt,
+ "completion_tokens": completion,
+ "total_tokens": prompt + completion,
+ "prompt_tokens_details": {"cached_tokens": 0},
+ },
+ }
+ )
+
+
+def _tc(id_, name, args):
+ return {
+ "id": id_,
+ "type": "function",
+ "function": {"name": name, "arguments": json.dumps(args)},
+ }
+
+
+class TestOpenAIChatCompletionModel:
+ def _model(self, **kw):
+ return OpenAIChatCompletionModel(model_name="gpt-test", api_key="sk-test", **kw)
+
+ def test_init(self):
+ model = self._model(base_url="http://router/v1")
+ assert model.client.base_url.host == "router" and model.client.api_key == "sk-test"
+ assert model.action_space_as_tools is True
+ assert (model.input_cost, model.output_cost) == (1e-3, 2e-3)
+
+ @pytest.mark.xfail(
+ strict=True,
+ reason="BUG: OpenAIChatCompletionModel.__init__ does not forward api_key to super()",
+ )
+ def test_init_keeps_api_key_attribute(self):
+ assert self._model().api_key == "sk-test"
+
+ def test_format_tools_for_chat_completion(self):
+ tools = [
+ {
+ "type": "function",
+ "name": "click",
+ "description": "d",
+ "parameters": {"p": 1},
+ "strict": True,
+ }
+ ]
+ assert OpenAIChatCompletionModel.format_tools_for_chat_completion(tools) == [
+ {
+ "type": "function",
+ "function": {"name": "click", "description": "d", "parameters": {"p": 1}},
+ }
+ ]
+ assert OpenAIChatCompletionModel.format_tools_for_chat_completion(None) is None
+
+ @pytest.mark.parametrize(
+ "payload_kw, expected_choice",
+ [
+ ({}, None),
+ ({"tool_choice": "none"}, "none"),
+ ({"tool_choice": "any"}, "required"),
+ ({"force_call_tool": "f"}, {"type": "function", "function": {"name": "f"}}),
+ ],
+ )
+ def test_call_api_params(self, payload_kw, expected_choice):
+ model = self._model(temperature=0.0, max_tokens=33)
+ tools = [{"type": "function", "name": "f", "description": "d", "parameters": {}}]
+ msgs = [OpenAIChatCompletionAPIMessageBuilder.user().add_text("hi").add_image("u")]
+ with patch.object(ra, "call_openai_api_with_retries", return_value="RAW") as call:
+ model._call_api(APIPayload(messages=msgs, tools=tools, **payload_kw))
+ fn, params = call.call_args.args
+ assert fn == model.client.chat.completions.create
+ assert params["messages"] == [
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "hi"},
+ {"type": "image_url", "image_url": {"url": "u"}},
+ ],
+ }
+ ]
+ assert params["temperature"] == 0.0 and params["max_completion_tokens"] == 33
+ assert params["tools"][0]["function"]["name"] == "f"
+ assert params.get("tool_choice") == expected_choice
+
+ def test_call_api_minimal(self):
+ model = self._model(max_tokens=None)
+ with patch.object(ra, "call_openai_api_with_retries", return_value="RAW") as call:
+ model._call_api(APIPayload(messages=[]))
+ assert call.call_args.args[1] == {"model": "gpt-test", "messages": []}
+
+ def test_parse_multiple_tool_calls(self):
+ response = _chat_completion(
+ {
+ "role": "assistant",
+ "content": "thinking",
+ "tool_calls": [_tc("c1", "click", {"bid": "a"}), _tc("c2", "noop", {})],
+ }
+ )
+ out = self._model()._parse_response(response)
+ assert out.action == "click(bid='a')\nnoop()"
+ assert out.tool_calls.tool_calls[0].raw_call["id"] == "c1"
+ assert out.tool_calls.raw_calls is response
+ assert out.think == "thinking"
+
+ def test_parse_no_tool_calls(self):
+ out = self._model()._parse_response(
+ _chat_completion({"role": "assistant", "content": "hi"})
+ )
+ assert out.action is None and out.tool_calls is None and out.think == "hi"
+
+ def test_text_action_space_returns_none(self):
+ model = self._model()
+ model.action_space_as_tools = False
+ response = _chat_completion(
+ {"role": "assistant", "content": "", "tool_calls": [_tc("c", "a", {})]}
+ )
+ assert model._parse_response(response).action is None
+
+ def test_thinking_wraps_reasoning_dict_message(self):
+ response = _ns(choices=[_ns(message={"reasoning": "why", "text": "T", "content": "C"})])
+ model = self._model()
+ assert model._extract_thinking_content_from_response(response) == "why\nTC"
+ assert (
+ model._extract_thinking_content_from_response(response, wrap_tag="r")
+ == "why\nTC"
+ )
+ empty = _ns(choices=[_ns(message={"reasoning": ""})])
+ assert model._extract_thinking_content_from_response(empty) == ""
+
+ @pytest.mark.xfail(
+ strict=True,
+ reason="BUG: tool-call-only chat responses have content=None, which is rendered as the "
+ "literal string 'None' in the extracted thinking",
+ )
+ def test_thinking_ignores_null_content(self):
+ response = _chat_completion(
+ {"role": "assistant", "content": None, "tool_calls": [_tc("c", "a", {})]}
+ )
+ assert self._model()._parse_response(response).think == ""
+
+ def test_tool_response_round_trip(self):
+ model = self._model()
+ response = _chat_completion(
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [_tc("call_9", "click", {"bid": "b"})],
+ }
+ )
+ calls = model._parse_response(response).tool_calls
+ for c in calls:
+ c.response_text("clicked")
+ out = OpenAIChatCompletionAPIMessageBuilder.add_responded_tool_calls(
+ calls
+ ).prepare_message()
+ assert out[0] is response.choices[0].message
+ assert out[1] == {
+ "name": "click",
+ "role": "tool",
+ "tool_call_id": "call_9",
+ "content": "clicked",
+ }
+
+
+# --------------------------------------------------------------------------- Claude model
+
+
+def _claude_response(blocks, inp=10, out=5):
+ usage = _ns(
+ input_tokens=inp, output_tokens=out, cache_input_tokens=0, cache_creation_input_tokens=0
+ )
+ return _ns(content=blocks, usage=usage)
+
+
+class TestClaudeResponseModel:
+ def _model(self, **kw):
+ return ClaudeResponseModel(model_name="claude-test", api_key="sk-ant", **kw)
+
+ def test_init(self):
+ model = self._model(base_url="http://claude.local")
+ assert model.client.api_key == "sk-ant" and "claude.local" in str(model.client.base_url)
+ assert model._pricing_api == "anthropic"
+ assert (model.input_cost, model.output_cost) == (1e-3, 2e-3)
+
+ def test_filter_system_messages(self):
+ sys1 = AnthropicAPIMessageBuilder.system().add_text("a")
+ user = AnthropicAPIMessageBuilder.user().add_text("u")
+ raw = {"role": "system", "content": "dict messages are not filtered"}
+ sys_msgs, others = ClaudeResponseModel.filter_system_messages([sys1, user, raw])
+ assert sys_msgs == [sys1] and others == [user, raw]
+
+ def test_filter_system_messages_rejects_typed_image(self):
+ sys_msg = AnthropicAPIMessageBuilder.system()
+ sys_msg.content.append({"type": "image"})
+ with pytest.raises(TypeError, match="cannot contain images"):
+ ClaudeResponseModel.filter_system_messages([sys_msg])
+
+ @pytest.mark.xfail(
+ strict=True,
+ reason="BUG: filter_system_messages checks c.get('type') but builder content is "
+ "{'image': ...}, so images added via add_image are never rejected",
+ )
+ def test_filter_system_messages_rejects_builder_image(self):
+ sys_msg = AnthropicAPIMessageBuilder.system().add_text("a").add_image("data:x")
+ with pytest.raises(TypeError):
+ ClaudeResponseModel.filter_system_messages([sys_msg])
+
+ @pytest.mark.parametrize(
+ "payload_kw, expected_choice",
+ [
+ ({}, None),
+ ({"tool_choice": "auto"}, {"type": "auto"}),
+ ({"tool_choice": "required"}, {"type": "any"}),
+ ({"tool_choice": "any"}, {"type": "any"}),
+ ({"force_call_tool": "click"}, {"type": "tool", "name": "click"}),
+ ],
+ )
+ def test_call_api_params(self, payload_kw, expected_choice):
+ model = self._model(temperature=0.7, max_tokens=50)
+ msgs = [
+ AnthropicAPIMessageBuilder.system().add_text("s1"),
+ AnthropicAPIMessageBuilder.system().add_text("s2"),
+ AnthropicAPIMessageBuilder.user().add_text("q"),
+ ]
+ tools = [{"name": "click"}]
+ with patch.object(ra, "call_anthropic_api_with_retries", return_value="RAW") as call:
+ assert model._call_api(APIPayload(messages=msgs, tools=tools, **payload_kw)) == "RAW"
+ fn, params = call.call_args.args
+ assert fn == model.client.messages.create
+ assert params["system"] == "s1\ns2"
+ assert params["messages"] == [{"role": "user", "content": [{"type": "text", "text": "q"}]}]
+ assert params["temperature"] == 0.7 and params["max_tokens"] == 50
+ assert params["tools"] == tools
+ assert params.get("tool_choice") == expected_choice
+
+ def test_call_api_minimal(self):
+ model = self._model(max_tokens=None)
+ with patch.object(ra, "call_anthropic_api_with_retries", return_value="RAW") as call:
+ model._call_api(APIPayload(messages=[]))
+ assert call.call_args.args[1] == {"model": "claude-test", "messages": [], "system": ""}
+
+ def test_cache_breakpoints_and_flags(self):
+ model = self._model()
+ first = AnthropicAPIMessageBuilder.user().add_text("old1").add_text("old2")
+ first.mark_all_previous_msg_for_caching()
+ second = AnthropicAPIMessageBuilder.user().add_text("new")
+ tools = [{"name": "a"}, {"name": "b"}]
+ payload = APIPayload(
+ messages=[first, second],
+ tools=tools,
+ use_cache_breakpoints=True,
+ cache_tool_definition=True,
+ cache_complete_prompt=True,
+ )
+ with patch.object(ra, "call_anthropic_api_with_retries", return_value="RAW") as call:
+ model._call_api(payload)
+ params = call.call_args.args[1]
+ eph = {"type": "ephemeral"}
+ assert "cache_control" not in params["messages"][0]["content"][0]
+ assert params["messages"][0]["content"][1]["cache_control"] == eph
+ assert params["messages"][1]["content"][0]["cache_control"] == eph
+ assert "cache_control" not in params["tools"][0]
+ assert params["tools"][1]["cache_control"] == eph
+
+ def test_breakpoint_ignored_when_disabled(self):
+ msg = AnthropicAPIMessageBuilder.user().add_text("x")
+ msg.mark_all_previous_msg_for_caching()
+ with patch.object(ra, "call_anthropic_api_with_retries", return_value="RAW") as call:
+ self._model()._call_api(APIPayload(messages=[msg]))
+ assert "cache_control" not in call.call_args.args[1]["messages"][0]["content"][0]
+
+ def test_apply_cache_breakpoints_without_marker(self):
+ prepared = [{"content": [{"type": "text", "text": "x"}]}]
+ assert self._model().apply_cache_breakpoints(object(), prepared) == [
+ {"content": [{"type": "text", "text": "x"}]}
+ ]
+
+ def test_parse_response(self):
+ tool1 = _ns(type="tool_use", name="click", input={"bid": "z"}, id="t1")
+ tool2 = _ns(type="tool_use", name="noop", input={}, id="t2")
+ response = _claude_response(
+ [_ns(type="text", text="I will "), _ns(type="text", text="click"), tool1, tool2]
+ )
+ out = self._model()._parse_response(response)
+ assert out.think == "I will click"
+ assert out.action == "click(bid='z')\nnoop()"
+ assert out.tool_calls.tool_calls[0].raw_call is tool1
+
+ def test_parse_single_and_none(self):
+ model = self._model()
+ one = model._parse_response(
+ _claude_response([_ns(type="tool_use", name="a", input={}, id="t")])
+ )
+ assert one.action == "a()" and one.think == ""
+ none = model._parse_response(_claude_response([_ns(type="text", text="no tools")]))
+ assert none.action is None and len(none.tool_calls) == 0
+
+ def test_text_action_space_returns_none(self):
+ model = self._model()
+ model.action_space_as_tools = False
+ out = model._parse_response(
+ _claude_response([_ns(type="tool_use", name="a", input={}, id="t")])
+ )
+ assert out.action is None
+
+ def test_full_call_tracks_effective_cost(self):
+ model = self._model()
+ response = _claude_response([_ns(type="tool_use", name="a", input={}, id="t")], 1000, 100)
+ response.usage = AnthropicUsage(
+ input_tokens=1000,
+ output_tokens=100,
+ cache_creation_input_tokens=0,
+ cache_read_input_tokens=0,
+ )
+ with patch.object(ra, "call_anthropic_api_with_retries", return_value=response):
+ with tracking.set_tracker() as tracker:
+ out = model(APIPayload(messages=[AnthropicAPIMessageBuilder.user().add_text("q")]))
+ assert out.action == "a()"
+ assert tracker.cost == pytest.approx(1.2)
+ assert model.stats.stats_dict["effective_cost"] == pytest.approx(1.2)
+
+
+# --------------------------------------------------------------------------- ModelArgs factories
+
+
+@pytest.mark.parametrize(
+ "args_cls, model_cls, builder_cls, model_name",
+ [
+ (OpenAIResponseModelArgs, OpenAIResponseModel, OpenAIResponseAPIMessageBuilder, "gpt-test"),
+ (
+ OpenAIChatModelArgs,
+ OpenAIChatCompletionModel,
+ OpenAIChatCompletionAPIMessageBuilder,
+ "gpt-test",
+ ),
+ (ClaudeResponseModelArgs, ClaudeResponseModel, AnthropicAPIMessageBuilder, "claude-test"),
+ ],
+)
+def test_model_args_factories(monkeypatch, args_cls, model_cls, builder_cls, model_name):
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-env")
+ monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-env")
+ args = args_cls(model_name=model_name, temperature=0.4, max_new_tokens=64)
+ model = args.make_model()
+ assert type(model) is model_cls
+ assert (model.model_name, model.temperature, model.max_tokens) == (model_name, 0.4, 64)
+ assert args.get_message_builder() is builder_cls
+ assert args.api in ("openai", "anthropic")
+
+
+def test_openrouter_model_args(monkeypatch):
+ monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or")
+ args = OpenRouterModelArgs(model_name="router/test", max_new_tokens=8)
+ model = args.make_model()
+ assert isinstance(model, OpenAIChatCompletionModel)
+ assert str(model.client.base_url).rstrip("/") == "https://openrouter.ai/api/v1"
+ assert model.client.api_key == "sk-or"
+ assert model.max_tokens == 8 and args.api == "openai"
+ assert args.get_message_builder() is OpenAIChatCompletionAPIMessageBuilder
diff --git a/tests/llm/test_tracking_unit.py b/tests/llm/test_tracking_unit.py
new file mode 100644
index 000000000..267c3cc06
--- /dev/null
+++ b/tests/llm/test_tracking_unit.py
@@ -0,0 +1,732 @@
+"""Hermetic unit tests for agentlab.llm.tracking (no network, no API keys)."""
+
+import importlib.util
+import logging
+import sys
+import threading
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+import agentlab.llm.tracking as tracking
+from agentlab.llm.tracking import (
+ ANTHROPIC_CACHE_PRICING_FACTOR,
+ OPENAI_CACHE_PRICING_FACTOR,
+ LLMTracker,
+ Stats,
+ TrackAPIPricingMixin,
+ set_tracker,
+)
+
+
+@pytest.fixture(autouse=True)
+def _reset_thread_local_tracker():
+ previous = getattr(tracking.TRACKER, "instance", None)
+ tracking.TRACKER.instance = None
+ tracking.get_pricing_openrouter.cache_clear()
+ yield
+ tracking.TRACKER.instance = previous
+ tracking.get_pricing_openrouter.cache_clear()
+
+
+class FakePricedModel(TrackAPIPricingMixin):
+ """Minimal concrete user of the mixin with a canned raw response."""
+
+ def __init__(self, model_name="fake-model", pricing_api="openai", response=None):
+ self.model_name = model_name
+ self._pricing_api = pricing_api
+ self.input_cost = 0.0
+ self.output_cost = 0.0
+ self.reset_stats()
+ self._response = response
+ self.calls = []
+
+ def _call_api(self, *args, **kwargs):
+ self.calls.append((args, kwargs))
+ return self._response
+
+ def _parse_response(self, response):
+ return ("parsed", response)
+
+
+def _priced(pricing_api, input_cost, output_cost):
+ model = FakePricedModel(pricing_api=pricing_api)
+ model.input_cost = input_cost
+ model.output_cost = output_cost
+ return model
+
+
+# --------------------------------------------------------------------------- LLMTracker
+
+
+class TestLLMTracker:
+ def test_default_keys_and_accumulation(self):
+ tracker = LLMTracker()
+ tracker(10, 5, 0.25)
+ tracker(1, 2, 0.5)
+ assert tracker.stats == {"input_tokens": 11, "output_tokens": 7, "cost": 0.75}
+
+ def test_suffix_changes_stat_keys(self):
+ tracker = LLMTracker(suffix="judge")
+ tracker(3, 4, 1.5)
+ assert tracker.stats == {
+ "input_tokens_judge": 3,
+ "output_tokens_judge": 4,
+ "cost_judge": 1.5,
+ }
+
+ def test_add_tracker_merges_counts(self):
+ a, b = LLMTracker(), LLMTracker("x")
+ a(1, 1, 1.0)
+ b(2, 3, 0.5)
+ a.add_tracker(b)
+ assert (a.input_tokens, a.output_tokens, a.cost) == (3, 4, 1.5)
+ # the merged-in tracker is untouched
+ assert (b.input_tokens, b.output_tokens, b.cost) == (2, 3, 0.5)
+
+ def test_repr(self):
+ tracker = LLMTracker()
+ tracker(2, 3, 0.125)
+ assert repr(tracker) == "LLMTracker(input_tokens=2, output_tokens=3, cost=0.125)"
+
+
+# --------------------------------------------------------------------------- set_tracker
+
+
+class TestSetTracker:
+ def test_installs_and_restores_none(self):
+ with set_tracker() as tracker:
+ assert tracking.TRACKER.instance is tracker
+ assert tracking.TRACKER.instance is None
+
+ def test_creates_instance_attribute_when_missing(self):
+ del tracking.TRACKER.instance
+ with set_tracker("s") as tracker:
+ assert tracker.cost_key == "cost_s"
+ assert tracking.TRACKER.instance is None
+
+ def test_nested_trackers_propagate_to_parent(self):
+ with set_tracker() as outer:
+ outer(1, 1, 1.0)
+ with set_tracker("inner") as inner:
+ inner(10, 20, 2.0)
+ assert tracking.TRACKER.instance is inner
+ assert tracking.TRACKER.instance is outer
+ assert inner.stats == {
+ "input_tokens_inner": 10,
+ "output_tokens_inner": 20,
+ "cost_inner": 2.0,
+ }
+ assert (outer.input_tokens, outer.output_tokens, outer.cost) == (11, 21, 3.0)
+
+ def test_restores_previous_on_exception_and_still_merges(self):
+ with set_tracker() as outer:
+ with pytest.raises(RuntimeError):
+ with set_tracker() as inner:
+ inner(5, 5, 0.5)
+ raise RuntimeError("boom")
+ assert tracking.TRACKER.instance is outer
+ assert outer.cost == 0.5
+
+ def test_threads_do_not_bleed(self):
+ barrier = threading.Barrier(2)
+ results = {}
+ errors = []
+
+ def worker(name, n_calls, cost):
+ try:
+ with set_tracker() as tracker:
+ barrier.wait(timeout=5) # both trackers are alive at the same time
+ for _ in range(n_calls):
+ tracking.TRACKER.instance(1, 2, cost)
+ barrier.wait(timeout=5)
+ results[name] = (tracker.stats, tracking.TRACKER.instance is tracker)
+ results[name + "_after"] = tracking.TRACKER.instance
+ except Exception as e: # pragma: no cover - surfaced below
+ errors.append(e)
+
+ threads = [
+ threading.Thread(target=worker, args=("a", 3, 1.0)),
+ threading.Thread(target=worker, args=("b", 5, 10.0)),
+ ]
+ with set_tracker() as main_tracker:
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join()
+ assert not errors
+ assert results["a"] == ({"input_tokens": 3, "output_tokens": 6, "cost": 3.0}, True)
+ assert results["b"] == ({"input_tokens": 5, "output_tokens": 10, "cost": 50.0}, True)
+ assert results["a_after"] is None and results["b_after"] is None
+ assert main_tracker.stats == {"input_tokens": 0, "output_tokens": 0, "cost": 0.0}
+
+
+# --------------------------------------------------------------------------- decorator
+
+
+def test_cost_tracker_decorator_with_suffix_updates_stats():
+ class Agent:
+ def get_action(self, obs):
+ tracking.TRACKER.instance(7, 3, 0.75)
+ return f"act({obs})", {"stats": {"existing": 1}}
+
+ wrapped = tracking.cost_tracker_decorator(Agent.get_action, suffix="sub")
+ action, info = wrapped(Agent(), "o")
+ assert action == "act(o)"
+ assert info["stats"] == {
+ "existing": 1,
+ "input_tokens_sub": 7,
+ "output_tokens_sub": 3,
+ "cost_sub": 0.75,
+ }
+ assert tracking.TRACKER.instance is None
+
+
+# --------------------------------------------------------------------------- pricing getters
+
+
+class TestGetPricingOpenRouter:
+ def test_parses_and_casts_pricing(self, monkeypatch):
+ monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test")
+ response = MagicMock(status_code=200)
+ response.json.return_value = {
+ "data": [
+ {"id": "a/model-1", "pricing": {"prompt": "0.000001", "completion": "0.000002"}},
+ {"id": "b/model-2", "pricing": {"prompt": "0", "completion": "1"}},
+ ]
+ }
+ with patch.object(tracking.requests, "get", return_value=response) as get:
+ prices = tracking.get_pricing_openrouter()
+ # second call is served from functools.cache
+ assert tracking.get_pricing_openrouter() is prices
+ get.assert_called_once_with(
+ "https://openrouter.ai/api/v1/models", headers={"Authorization": "Bearer sk-test"}
+ )
+ assert prices == {
+ "a/model-1": {"prompt": 1e-06, "completion": 2e-06},
+ "b/model-2": {"prompt": 0.0, "completion": 1.0},
+ }
+
+ def test_non_200_raises(self, monkeypatch):
+ monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test")
+ with patch.object(tracking.requests, "get", return_value=MagicMock(status_code=500)):
+ with pytest.raises(ValueError, match="Failed to get model metadata"):
+ tracking.get_pricing_openrouter()
+
+ def test_missing_key_asserts(self, monkeypatch):
+ monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+ with patch.object(tracking.requests, "get") as get:
+ with pytest.raises(AssertionError, match="OpenRouter API key is required"):
+ tracking.get_pricing_openrouter()
+ get.assert_not_called()
+
+
+class TestGetPricingOpenAI:
+ def test_pairs_prompt_and_completion_per_token(self, monkeypatch):
+ fake = SimpleNamespace(
+ MODEL_COST_PER_1K_TOKENS={
+ "gpt-x": 1.0,
+ "gpt-x-completion": 2.0,
+ "orphan": 5.0, # no completion entry -> dropped
+ "lonely-completion": 3.0, # no prompt entry -> dropped
+ }
+ )
+ monkeypatch.setattr(tracking, "openai_info", fake)
+ assert tracking.get_pricing_openai() == {"gpt-x": {"prompt": 0.001, "completion": 0.002}}
+
+ def test_missing_langchain_returns_empty(self, monkeypatch, caplog):
+ monkeypatch.setattr(tracking, "openai_info", None)
+ with caplog.at_level(logging.WARNING):
+ assert tracking.get_pricing_openai() == {}
+ assert "Failed to get OpenAI pricing" in caplog.text
+
+
+@pytest.mark.parametrize(
+ "name, expected",
+ [
+ ("anthropic.claude-v2", "claude"),
+ ("anthropic.claude-v2:1", "claude"),
+ ("anthropic.claude-3-sonnet-20240229-v1:0", "claude-3-sonnet-20240229"),
+ ("anthropic.claude-instant-v1", "claude-instant"),
+ ("claude-3-haiku-v1.5", "claude-3-haiku"),
+ ("claude-3-haiku", "claude-3-haiku"),
+ ("model-v2-mini", "model-v2-mini"), # version only stripped at the end
+ ],
+)
+def test_remove_version_suffix(name, expected):
+ assert tracking._remove_version_suffix(name) == expected
+
+
+class TestGetPricingAnthropic:
+ def test_merges_input_and_output_tables(self, monkeypatch):
+ fake = SimpleNamespace(
+ MODEL_COST_PER_1K_INPUT_TOKENS={
+ "anthropic.claude-v2": 8.0,
+ "anthropic.claude-3-v1:0": 3.0,
+ },
+ MODEL_COST_PER_1K_OUTPUT_TOKENS={
+ "anthropic.claude-v2": 24.0,
+ "anthropic.claude-3-v1:0": 15.0,
+ "anthropic.output-only-v1": 1.0,
+ },
+ )
+ monkeypatch.setattr(tracking, "bedrock_anthropic_callback", fake)
+ assert tracking.get_pricing_anthropic() == {
+ "claude": {"prompt": 0.008, "completion": 0.024},
+ "claude-3": {"prompt": 0.003, "completion": 0.015},
+ "output-only": {"completion": 0.001},
+ }
+
+ def test_missing_langchain_returns_empty(self, monkeypatch, caplog):
+ monkeypatch.setattr(tracking, "bedrock_anthropic_callback", None)
+ with caplog.at_level(logging.WARNING):
+ assert tracking.get_pricing_anthropic() == {}
+ assert "Failed to get Anthropic pricing" in caplog.text
+
+
+class TestGetPricingLiteLLM:
+ def test_uses_model_info(self):
+ info = {"input_cost_per_token": 1e-6, "output_cost_per_token": 4e-6}
+ with patch.object(tracking, "get_model_info", return_value=info) as gmi:
+ assert tracking.get_pricing_litellm("m") == {"m": {"prompt": 1e-6, "completion": 4e-6}}
+ gmi.assert_called_once_with("m")
+
+ def test_partial_info_defaults_to_zero(self):
+ with patch.object(tracking, "get_model_info", return_value={"input_cost_per_token": 2.0}):
+ assert tracking.get_pricing_litellm("m") == {"m": {"prompt": 2.0, "completion": 0.0}}
+
+ def test_error_logged_and_zero(self, caplog):
+ with patch.object(tracking, "get_model_info", side_effect=Exception("unknown model")):
+ with caplog.at_level(logging.ERROR):
+ assert tracking.get_pricing_litellm("m") == {
+ "m": {"prompt": 0.0, "completion": 0.0}
+ }
+ assert "unknown model" in caplog.text
+
+
+# --------------------------------------------------------------------------- mixin: pricing setup
+
+
+class TestPricingSetup:
+ @pytest.mark.parametrize(
+ "api, attr",
+ [
+ ("openai", "get_pricing_openai"),
+ ("anthropic", "get_pricing_anthropic"),
+ ("openrouter", "get_pricing_openrouter"),
+ ],
+ )
+ def test_fetch_dispatches_per_provider(self, monkeypatch, api, attr):
+ sentinel = {"fake-model": {"prompt": 1, "completion": 2}}
+ monkeypatch.setattr(tracking, attr, lambda: sentinel)
+ model = FakePricedModel(pricing_api=api)
+ assert model.fetch_pricing_information_from_provider() is sentinel
+
+ def test_fetch_litellm_passes_model_name(self, monkeypatch):
+ seen = []
+ monkeypatch.setattr(tracking, "get_pricing_litellm", lambda name: seen.append(name) or {})
+ FakePricedModel(
+ model_name="lite/m", pricing_api="litellm"
+ ).fetch_pricing_information_from_provider()
+ assert seen == ["lite/m"]
+
+ def test_fetch_unsupported_returns_none(self, caplog):
+ model = FakePricedModel(pricing_api="vllm")
+ with caplog.at_level(logging.WARNING):
+ assert model.fetch_pricing_information_from_provider() is None
+ assert "Unsupported provider: vllm" in caplog.text
+
+ def test_init_pricing_tracker_uses_provider_table(self, monkeypatch):
+ monkeypatch.setattr(
+ tracking,
+ "get_pricing_openai",
+ lambda: {"fake-model": {"prompt": "0.5", "completion": 2}},
+ )
+ model = FakePricedModel(pricing_api=None)
+ model.stats.increment_stats_dict({"x": 1})
+ model.init_pricing_tracker(pricing_api="openai")
+ assert model._pricing_api == "openai"
+ assert (model.input_cost, model.output_cost) == (0.5, 2.0)
+ assert isinstance(model.input_cost, float)
+ assert dict(model.stats.stats_dict) == {} # stats are reset
+
+ def test_falls_back_to_litellm_model_info(self, monkeypatch):
+ monkeypatch.setattr(tracking, "get_pricing_openai", lambda: {"other": {}})
+ info = {"input_cost_per_token": 3e-6, "output_cost_per_token": 6e-6}
+ with patch.object(tracking, "get_model_info", return_value=info):
+ model = FakePricedModel(pricing_api="openai")
+ model.set_pricing_attributes()
+ assert (model.input_cost, model.output_cost) == (3e-6, 6e-6)
+
+ def test_unsupported_provider_falls_back_to_litellm(self):
+ with patch.object(tracking, "get_model_info", return_value={}):
+ model = FakePricedModel(pricing_api="nope")
+ model.set_pricing_attributes()
+ assert (model.input_cost, model.output_cost) == (0.0, 0.0)
+
+ def test_fallback_failure_zeroes_costs(self, monkeypatch, caplog):
+ monkeypatch.setattr(tracking, "get_pricing_openai", lambda: {})
+ model = FakePricedModel(pricing_api="openai")
+ model.input_cost, model.output_cost = 9.0, 9.0
+ with patch.object(tracking, "get_model_info", side_effect=Exception("not mapped")):
+ with caplog.at_level(logging.WARNING):
+ model.set_pricing_attributes()
+ assert (model.input_cost, model.output_cost) == (0.0, 0.0)
+ assert "Failed to fetch pricing for fake-model" in caplog.text
+
+
+# --------------------------------------------------------------------------- token counts
+
+
+class TestTokenCounts:
+ @pytest.mark.parametrize(
+ "response, expected",
+ [
+ (SimpleNamespace(usage=SimpleNamespace(input_tokens=3, output_tokens=4)), (3, 4)),
+ (SimpleNamespace(usage=SimpleNamespace(prompt_tokens=5, completion_tokens=6)), (5, 6)),
+ ({"usage": {"input_tokens": 7, "output_tokens": 8}}, (7, 8)),
+ ({"usage": {"prompt_tokens": 9, "completion_tokens": 10}}, (9, 10)),
+ ],
+ ids=["obj-responses", "obj-chat", "dict-responses", "dict-chat"],
+ )
+ def test_extraction(self, response, expected):
+ assert FakePricedModel().get_tokens_counts_from_response(response) == expected
+
+ @pytest.mark.parametrize(
+ "response",
+ [
+ SimpleNamespace(usage=None),
+ SimpleNamespace(usage=SimpleNamespace(input_tokens=3)),
+ {"usage": {"input_tokens": 1}},
+ {"no_usage": True},
+ object(),
+ ],
+ ids=["usage-none", "obj-missing-output", "dict-missing-output", "dict-no-usage", "bare"],
+ )
+ def test_unextractable_defaults_to_zero(self, response, caplog):
+ with caplog.at_level(logging.WARNING):
+ assert FakePricedModel().get_tokens_counts_from_response(response) == (0, 0)
+ assert "Unable to extract input and output tokens" in caplog.text
+
+ def test_update_pricing_tracker_feeds_active_tracker(self):
+ model = _priced("openai", 0.5, 2.0)
+ response = SimpleNamespace(usage=SimpleNamespace(input_tokens=10, output_tokens=3))
+ with set_tracker() as tracker:
+ model.update_pricing_tracker(response)
+ assert tracker.stats == {"input_tokens": 10, "output_tokens": 3, "cost": 11.0}
+
+ def test_update_pricing_tracker_without_tracker_is_noop(self):
+ model = _priced("openai", 0.5, 2.0)
+ response = SimpleNamespace(usage=SimpleNamespace(input_tokens=10, output_tokens=3))
+ model.update_pricing_tracker(response) # TRACKER.instance is None
+ del tracking.TRACKER.instance
+ model.update_pricing_tracker(response) # attribute missing entirely
+ assert not hasattr(tracking.TRACKER, "instance")
+
+
+# --------------------------------------------------------------------------- effective cost
+
+
+def _anthropic_usage(input_tokens, output_tokens, cache_read, cache_write):
+ # attribute names as read by the current implementation
+ return SimpleNamespace(
+ input_tokens=input_tokens,
+ output_tokens=output_tokens,
+ cache_input_tokens=cache_read,
+ cache_creation_input_tokens=cache_write,
+ )
+
+
+class TestEffectiveCostAnthropic:
+ @pytest.mark.parametrize(
+ "usage, expected",
+ [
+ # input*1e-3 + output*5e-3 + read*1e-3*0.1 + write*1e-3*1.25
+ (_anthropic_usage(1000, 200, 0, 0), 1.0 + 1.0),
+ (_anthropic_usage(1000, 200, 4000, 0), 1.0 + 1.0 + 0.4),
+ (_anthropic_usage(1000, 200, 0, 800), 1.0 + 1.0 + 1.0),
+ (_anthropic_usage(100, 0, 2000, 400), 0.1 + 0.2 + 0.5),
+ (SimpleNamespace(), 0.0), # all attributes default to 0
+ ],
+ ids=["no-cache", "cache-read", "cache-write", "read+write", "empty-usage"],
+ )
+ def test_table(self, usage, expected):
+ model = _priced("anthropic", 1e-3, 5e-3)
+ cost = model.get_effective_cost(SimpleNamespace(usage=usage))
+ assert cost == pytest.approx(expected, rel=1e-12)
+
+ def test_factors(self):
+ assert ANTHROPIC_CACHE_PRICING_FACTOR == {
+ "cache_read_tokens": 0.1,
+ "cache_write_tokens": 1.25,
+ }
+
+ def test_no_usage_attribute_costs_zero(self):
+ assert _priced("anthropic", 1.0, 1.0).get_effective_cost(object()) == 0.0
+
+ def test_negative_cost_warns(self, caplog):
+ model = _priced("anthropic", -1.0, 0.0)
+ with caplog.at_level(logging.WARNING):
+ cost = model.get_effective_cost(SimpleNamespace(usage=_anthropic_usage(1, 0, 0, 0)))
+ assert cost == -1.0
+ assert "Negative effective cost" in caplog.text
+
+ @pytest.mark.xfail(
+ strict=True,
+ reason="BUG: cache reads are read from usage.cache_input_tokens; Anthropic's field is "
+ "cache_read_input_tokens, so cache-read tokens are never billed",
+ )
+ def test_real_anthropic_usage_bills_cache_reads(self):
+ from anthropic.types import Usage
+
+ usage = Usage(
+ input_tokens=1000,
+ output_tokens=200,
+ cache_read_input_tokens=4000,
+ cache_creation_input_tokens=0,
+ )
+ cost = _priced("anthropic", 1e-3, 5e-3).get_effective_cost(SimpleNamespace(usage=usage))
+ assert cost == pytest.approx(2.4, rel=1e-12)
+
+ @pytest.mark.xfail(
+ strict=True,
+ raises=TypeError,
+ reason="BUG: anthropic Usage leaves cache_*_input_tokens as None when unused; "
+ "None * float raises TypeError instead of costing 0",
+ )
+ def test_real_anthropic_usage_without_cache_fields(self):
+ from anthropic.types import Usage
+
+ usage = Usage(input_tokens=1000, output_tokens=200)
+ cost = _priced("anthropic", 1e-3, 5e-3).get_effective_cost(SimpleNamespace(usage=usage))
+ assert cost == pytest.approx(2.0, rel=1e-12)
+
+
+class _ChatUsage:
+ """Chat Completions-style usage object (has prompt_tokens_details)."""
+
+ def __init__(self, prompt, completion, cached):
+ self.prompt_tokens = prompt
+ self.completion_tokens = completion
+ self.prompt_tokens_details = (
+ None if cached is None else SimpleNamespace(cached_tokens=cached)
+ )
+
+
+class _ResponsesUsage:
+ """Responses API-style usage object (has input_tokens_details, no prompt_tokens_details)."""
+
+ def __init__(self, inp, out, cached):
+ self.input_tokens = inp
+ self.output_tokens = out
+ self.input_tokens_details = (
+ None if cached is None else SimpleNamespace(cached_tokens=cached)
+ )
+
+
+class TestEffectiveCostOpenAI:
+ @pytest.mark.parametrize(
+ "usage, expected",
+ [
+ # new_input*1e-3 + cached*1e-3*0.5 + output*4e-3
+ (_ChatUsage(1000, 250, 0), 1.0 + 1.0),
+ (_ChatUsage(1000, 250, 600), 0.4 + 0.3 + 1.0),
+ (_ChatUsage(1000, 250, None), 1.0 + 1.0),
+ (_ResponsesUsage(2000, 500, 0), 2.0 + 2.0),
+ (_ResponsesUsage(2000, 500, 2000), 0.0 + 1.0 + 2.0),
+ (_ResponsesUsage(2000, 500, None), 2.0 + 2.0),
+ ],
+ ids=[
+ "chat-no-cache",
+ "chat-cached",
+ "chat-details-none",
+ "responses-no-cache",
+ "responses-all-cached",
+ "responses-details-none",
+ ],
+ )
+ def test_table(self, usage, expected):
+ model = _priced("openai", 1e-3, 4e-3)
+ cost = model.get_effective_cost(SimpleNamespace(usage=usage))
+ assert cost == pytest.approx(expected, rel=1e-12)
+
+ def test_factors(self):
+ assert OPENAI_CACHE_PRICING_FACTOR == {"cache_read_tokens": 0.5, "cache_write_tokens": 1}
+
+ def test_real_sdk_usage_objects(self):
+ from openai.types.chat import ChatCompletion
+ from openai.types.responses import ResponseUsage
+
+ chat = ChatCompletion.model_validate(
+ {
+ "id": "x",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "m",
+ "choices": [],
+ "usage": {
+ "prompt_tokens": 100,
+ "completion_tokens": 10,
+ "total_tokens": 110,
+ "prompt_tokens_details": {"cached_tokens": 40},
+ },
+ }
+ )
+ responses_usage = ResponseUsage.model_validate(
+ {
+ "input_tokens": 100,
+ "output_tokens": 10,
+ "total_tokens": 110,
+ "input_tokens_details": {"cached_tokens": 40},
+ "output_tokens_details": {"reasoning_tokens": 0},
+ }
+ )
+ model = _priced("openai", 0.01, 0.1)
+ expected = 60 * 0.01 + 40 * 0.005 + 10 * 0.1
+ assert model.get_effective_cost(chat) == pytest.approx(expected, rel=1e-12)
+ assert model.get_effective_cost(SimpleNamespace(usage=responses_usage)) == pytest.approx(
+ expected, rel=1e-12
+ )
+
+ def test_missing_usage_costs_zero(self, caplog):
+ with caplog.at_level(logging.WARNING):
+ assert _priced("openai", 1.0, 1.0).get_effective_cost(SimpleNamespace()) == 0.0
+ assert "No usage information" in caplog.text
+
+ def test_negative_cost_warns(self, caplog):
+ model = _priced("openai", 1.0, 1.0)
+ with caplog.at_level(logging.WARNING):
+ # more cached tokens than prompt tokens -> negative new_input
+ cost = model.get_effective_cost(SimpleNamespace(usage=_ChatUsage(1, 0, 10)))
+ assert cost == pytest.approx(-9 + 5.0)
+ assert "OpenAI: Negative effective cost" in caplog.text
+
+
+class TestEffectiveCostDispatch:
+ def test_litellm_uses_completion_cost(self):
+ response = object()
+ with patch.object(tracking, "completion_cost", return_value=0.42) as cc:
+ assert _priced("litellm", 0, 0).get_effective_cost(response) == 0.42
+ cc.assert_called_once_with(response)
+
+ def test_unknown_provider_zero(self, caplog):
+ with caplog.at_level(logging.WARNING):
+ assert _priced("openrouter", 1, 1).get_effective_cost(object()) == 0.0
+ assert "No effective cost calculated" in caplog.text
+
+
+# --------------------------------------------------------------------------- mixin __call__ / Stats
+
+
+class TestMixinCall:
+ def test_chat_completion_response_updates_stats_and_tracker(self):
+ usage = _IterableUsage(
+ {
+ "prompt_tokens": 10,
+ "completion_tokens": 5,
+ "total_tokens": 15,
+ "prompt_tokens_details": SimpleNamespace(cached_tokens=4),
+ }
+ )
+ response = SimpleNamespace(usage=usage)
+ model = _priced("openai", 0.1, 1.0)
+ model._response = response
+ with set_tracker() as tracker:
+ out = model("a", k=1)
+ model("b")
+ assert out == ("parsed", response)
+ assert model.calls == [(("a",), {"k": 1}), (("b",), {})]
+ effective = (6 * 0.1 + 4 * 0.05 + 5 * 1.0) * 2
+ stats = dict(model.stats.stats_dict)
+ assert stats.pop("effective_cost") == pytest.approx(effective)
+ assert stats == {
+ "usage_prompt_tokens": 20,
+ "usage_completion_tokens": 10,
+ "usage_total_tokens": 30,
+ "usage_cached_tokens": 8,
+ "n_api_calls": 2,
+ }
+ assert tracker.input_tokens == 20 and tracker.output_tokens == 10
+ assert tracker.cost == pytest.approx(2 * (10 * 0.1 + 5 * 1.0))
+
+ def test_responses_api_cached_tokens(self):
+ usage = _IterableUsage(
+ {
+ "input_tokens": 8,
+ "output_tokens": 2,
+ "input_tokens_details": SimpleNamespace(cached_tokens=3),
+ "prompt_tokens_details": None,
+ }
+ )
+ model = FakePricedModel(pricing_api="anthropic", response=SimpleNamespace(usage=usage))
+ model("x")
+ stats = dict(model.stats.stats_dict)
+ assert stats["usage_cached_tokens"] == 3
+ assert stats["usage_input_tokens"] == 8
+ assert "usage_input_tokens_details" not in stats
+
+ def test_response_without_usage(self):
+ model = FakePricedModel(pricing_api="nope", response=SimpleNamespace())
+ assert model() == ("parsed", model._response)
+ assert dict(model.stats.stats_dict) == {"n_api_calls": 1, "effective_cost": 0.0}
+
+
+class _IterableUsage(SimpleNamespace):
+ """Usage object that supports dict(obj) like pydantic models and exposes attributes."""
+
+ def __init__(self, fields):
+ super().__init__(**fields)
+
+ def __iter__(self):
+ return iter(vars(self).items())
+
+
+def test_stats_increment_accumulates_and_defaults():
+ stats = Stats()
+ stats.increment_stats_dict({"a": 1, "b": 0.5})
+ stats.increment_stats_dict({"a": 2, "c": 3})
+ assert dict(stats.stats_dict) == {"a": 3, "b": 0.5, "c": 3}
+ assert stats.stats_dict["missing"] == 0.0
+ assert Stats().stats_dict is not stats.stats_dict
+
+
+# --------------------------------------------------------------------------- import fallbacks
+
+
+def _exec_tracking_copy(name):
+ path = Path(tracking.__file__)
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_import_without_langchain_community(monkeypatch):
+ real_find_spec = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a, **k: (
+ None if name == "langchain_community" else real_find_spec(name, *a, **k)
+ ),
+ )
+ module = _exec_tracking_copy("_tracking_copy_no_langchain")
+ assert module.openai_info is None and module.bedrock_anthropic_callback is None
+ assert module.get_pricing_openai() == {}
+
+
+def test_import_with_broken_langchain_callbacks(monkeypatch):
+ real_find_spec = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a, **k: (
+ object() if name == "langchain_community" else real_find_spec(name, *a, **k)
+ ),
+ )
+ monkeypatch.setitem(sys.modules, "langchain_community.callbacks", None)
+ module = _exec_tracking_copy("_tracking_copy_broken_langchain")
+ assert module.openai_info is None and module.bedrock_anthropic_callback is None
+ assert module.get_pricing_anthropic() == {}