diff --git a/tests/agents/test_tool_use_agent_unit.py b/tests/agents/test_tool_use_agent_unit.py
new file mode 100644
index 000000000..59d8a9ff4
--- /dev/null
+++ b/tests/agents/test_tool_use_agent_unit.py
@@ -0,0 +1,1094 @@
+"""Hermetic unit tests for agentlab.agents.tool_use_agent.tool_use_agent.
+
+The LLM is replaced by a scripted FakeModel; no network, browser or API key is used.
+"""
+
+import json
+from dataclasses import dataclass, field
+from types import SimpleNamespace
+from unittest import mock
+
+import numpy as np
+import pytest
+import requests
+
+from agentlab.agents.tool_use_agent import tool_use_agent as tua
+from agentlab.agents.tool_use_agent.tool_use_agent import (
+ AXTREE_NOTE,
+ DEFAULT_PROMPT_CONFIG,
+ SYS_MSG,
+ GeneralHints,
+ Goal,
+ MsgGroup,
+ Obs,
+ PromptConfig,
+ StructuredDiscussion,
+ Summarizer,
+ TaskHint,
+ ToolUseAgent,
+ ToolUseAgentArgs,
+ _format_tabs,
+)
+from agentlab.benchmarks.osworld import OSWorldActionSet
+from agentlab.llm.base_api import BaseModelArgs
+from agentlab.llm.response_api import (
+ AnthropicAPIMessageBuilder,
+ APIPayload,
+ LLMOutput,
+ MessageBuilder,
+ OpenAIResponseAPIMessageBuilder,
+ ToolCall,
+ ToolCalls,
+)
+from agentlab.llm.tracking import TRACKER
+
+# --------------------------------------------------------------------------------------
+# Fakes and helpers
+# --------------------------------------------------------------------------------------
+
+
+class FakeModel:
+ """Scripted stand-in for a response-API model: returns queued LLMOutputs in order."""
+
+ def __init__(self, responses=None, msg=None):
+ self.responses = list(responses or [])
+ self.payloads: list[APIPayload] = []
+ self.reset_count = 0
+ self.stats = SimpleNamespace(stats_dict={"n_api_calls": 0})
+ if msg is not None:
+ self.msg = msg
+
+ def reset_stats(self):
+ self.reset_count += 1
+ self.stats.stats_dict = {"n_api_calls": 0}
+
+ def __call__(self, payload: APIPayload) -> LLMOutput:
+ assert isinstance(payload, APIPayload)
+ # snapshot messages: the discussion list is mutated by the agent after the call
+ self.payloads.append(
+ SimpleNamespace(payload=payload, messages=list(payload.messages or []))
+ )
+ self.stats.stats_dict["n_api_calls"] += 1
+ if not self.responses:
+ raise AssertionError("FakeModel ran out of scripted responses")
+ return self.responses.pop(0)
+
+
+@dataclass
+class FakeModelArgs(BaseModelArgs):
+ model_name: str = "fake/model-1"
+ api: str = "openai"
+ builder: type = OpenAIResponseAPIMessageBuilder
+ responses: list = field(default_factory=list)
+ prepared: int = 0
+ closed: int = 0
+
+ def make_model(self):
+ self._model = FakeModel(self.responses)
+ return self._model
+
+ def get_message_builder(self):
+ return self.builder
+
+ def prepare_server(self):
+ self.prepared += 1
+ return "prepared"
+
+ def close_server(self):
+ self.closed += 1
+ return "closed"
+
+
+class FakeActionSet:
+ def __init__(self):
+ self.apis = []
+
+ def to_tool_description(self, api="openai"):
+ self.apis.append(api)
+ return [{"type": "function", "name": "click", "parameters": {}}]
+
+
+def make_tool_calls(*names):
+ calls = ToolCalls()
+ for i, name in enumerate(names):
+ calls.add_tool_call(
+ ToolCall(name=name, arguments={"x": i}, raw_call=SimpleNamespace(id=f"c{i}"))
+ )
+ return calls
+
+
+def llm_output(action="click(1)", think="thinking", tool_names=("click",)):
+ return LLMOutput(think=think, action=action, tool_calls=make_tool_calls(*tool_names))
+
+
+def fake_llm(builder=OpenAIResponseAPIMessageBuilder, responses=None):
+ return FakeModel(responses, msg=builder)
+
+
+def make_obs(**overrides):
+ obs = {
+ "goal_object": [{"type": "text", "text": "Book a flight to Paris"}],
+ "last_action_error": "",
+ "screenshot": np.zeros((4, 4, 3), dtype=np.uint8),
+ "axtree_txt": "[a1] button 'Go'",
+ "pruned_html": "",
+ "open_pages_urls": ["http://a.test", "http://b.test"],
+ "open_pages_titles": ["A", "B"],
+ "active_page_index": 1,
+ "elapsed_time": 1.5,
+ }
+ obs.update(overrides)
+ return obs
+
+
+def texts(msg: MessageBuilder) -> list[str]:
+ return [c["text"] for c in msg.content if "text" in c]
+
+
+def write_hint_db(path, rows):
+ header = "task_name,semantic_keys,hint\n"
+ body = "".join(f'{t},{k},"{h}"\n' for t, k, h in rows)
+ path.write_text(header + body)
+ return path
+
+
+HINT_ROWS = [
+ ("miniwob.book-flight", "dropdown", "select from the dropdown list"),
+ ("miniwob.book-flight", "forms", "activate the field first"),
+ ("miniwob.click-test", "dropdown", "select from the dropdown list"),
+ ("workarena.order", "forms", " use the form "),
+]
+
+
+@pytest.fixture
+def hint_csv(tmp_path):
+ return write_hint_db(tmp_path / "hints.csv", HINT_ROWS)
+
+
+@pytest.fixture(autouse=True)
+def _reset_tracker():
+ previous = getattr(TRACKER, "instance", None)
+ TRACKER.instance = None
+ yield
+ TRACKER.instance = previous
+
+
+def make_config(hint_path, **overrides):
+ cfg = dict(
+ goal=Goal(goal_as_system_msg=True),
+ obs=Obs(use_last_error=True, use_screenshot=True, use_axtree=True),
+ summarizer=Summarizer(do_summary=False),
+ general_hints=GeneralHints(use_hints=False),
+ task_hint=TaskHint(use_task_hint=False, hint_db_rel_path=str(hint_path)),
+ keep_last_n_obs=None,
+ multiaction=False,
+ action_subsets=("coord",),
+ )
+ cfg.update(overrides)
+ return PromptConfig(**cfg)
+
+
+def make_agent(hint_path, responses, builder=OpenAIResponseAPIMessageBuilder, **cfg):
+ model_args = FakeModelArgs(builder=builder, responses=list(responses))
+ action_set = FakeActionSet()
+ agent = ToolUseAgent(
+ model_args=model_args, config=make_config(hint_path, **cfg), action_set=action_set
+ )
+ agent.set_task_name("miniwob.book-flight")
+ return agent, agent.llm, action_set
+
+
+# --------------------------------------------------------------------------------------
+# Block.make
+# --------------------------------------------------------------------------------------
+
+
+def test_block_make_returns_independent_copy_and_calls_init():
+ original = Obs(use_axtree=True, use_tabs=True)
+ copy_ = original.make()
+ assert copy_ is not original
+ assert copy_ == original
+ copy_.use_axtree = False
+ assert original.use_axtree is True
+
+
+def test_block_make_runs_init_on_copy_only(hint_csv):
+ block = TaskHint(hint_db_rel_path=str(hint_csv))
+ made = block.make()
+ assert not hasattr(block, "hint_db")
+ assert list(made.hint_db.columns) == ["task_name", "semantic_keys", "hint"]
+
+
+def test_block_base_init_and_apply_are_noops():
+ assert Goal()._init() is None
+ with pytest.raises(TypeError):
+ tua.Block() # abstract
+
+
+# --------------------------------------------------------------------------------------
+# StructuredDiscussion
+# --------------------------------------------------------------------------------------
+
+
+def user_msg(text, builder=AnthropicAPIMessageBuilder):
+ return builder.user().add_text(text)
+
+
+def test_msg_group_defaults_are_not_shared():
+ a, b = MsgGroup("a"), MsgGroup("b")
+ a.messages.append("x")
+ assert b.messages == []
+ assert a.summary is None
+
+
+def test_new_group_auto_names_by_index():
+ d = StructuredDiscussion()
+ d.new_group("goal")
+ d.new_group()
+ d.new_group()
+ assert [g.name for g in d.groups] == ["goal", "group_1", "group_2"]
+
+
+def test_append_without_group_raises():
+ with pytest.raises(IndexError):
+ StructuredDiscussion().append(user_msg("x"))
+
+
+def test_goal_set_and_last_summary_on_empty_discussion():
+ d = StructuredDiscussion()
+ assert d.is_goal_set() is False
+ assert d.get_last_summary() is None
+ d.new_group()
+ assert d.is_goal_set() is True
+ assert d.get_last_summary() is None
+ summary = user_msg("sum")
+ d.set_last_summary(summary)
+ assert d.get_last_summary() is summary
+
+
+def test_flatten_without_limit_keeps_all_messages_in_order():
+ d = StructuredDiscussion(keep_last_n_obs=None)
+ msgs = []
+ for g in range(3):
+ d.new_group()
+ for j in range(2):
+ m = user_msg(f"{g}-{j}")
+ msgs.append(m)
+ d.append(m)
+ d.set_last_summary(user_msg(f"summary {g}"))
+ assert d.flatten() == msgs
+
+
+def test_flatten_replaces_old_groups_by_summary_and_keeps_tail():
+ d = StructuredDiscussion(keep_last_n_obs=1)
+ d.new_group("goal")
+ goal = user_msg("goal")
+ d.append(goal) # no summary -> always kept in full
+ d.new_group()
+ o1, s1 = user_msg("obs1"), user_msg("sum1")
+ d.append(o1)
+ d.set_last_summary(s1)
+ d.new_group()
+ o2, s2 = user_msg("obs2"), user_msg("sum2")
+ d.append(o2)
+ d.set_last_summary(s2)
+
+ flat = d.flatten()
+ assert flat == [goal, s1, o2] # the tail group keeps raw messages, not its summary
+
+
+def test_flatten_resets_previous_breakpoints_and_sets_a_new_one():
+ d = StructuredDiscussion(keep_last_n_obs=1)
+ msgs = []
+ for g in range(3):
+ d.new_group()
+ m = user_msg(f"m{g}")
+ m._cache_breakpoint = True # stale breakpoint from a previous step
+ d.append(m)
+ msgs.append(m)
+ flat = d.flatten()
+ assert flat == msgs
+ assert [m._cache_breakpoint for m in flat] == [False, False, True]
+
+
+def test_flatten_with_keep_larger_than_groups_marks_nothing():
+ d = StructuredDiscussion(keep_last_n_obs=5)
+ d.new_group()
+ m = user_msg("only")
+ d.append(m)
+ assert d.flatten() == [m]
+ assert not hasattr(m, "_cache_breakpoint")
+
+
+# --------------------------------------------------------------------------------------
+# Goal block
+# --------------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+ "builder, as_system, expected_role",
+ [
+ (OpenAIResponseAPIMessageBuilder, True, "developer"),
+ (OpenAIResponseAPIMessageBuilder, False, "user"),
+ (AnthropicAPIMessageBuilder, True, "system"),
+ (AnthropicAPIMessageBuilder, False, "user"),
+ ],
+)
+def test_goal_block_adds_system_and_goal_messages(builder, as_system, expected_role):
+ llm = fake_llm(builder)
+ d = StructuredDiscussion()
+ d.new_group("goal")
+ obs = make_obs(
+ goal_object=[
+ {"type": "text", "text": "Do X"},
+ {"type": "image_url", "image_url": "data:image/png;base64,AAA"},
+ {"type": "unknown", "foo": 1},
+ ]
+ )
+ Goal(goal_as_system_msg=as_system).apply(llm, d, obs, sys_msg="SYSTEM")
+ sys_msg, goal_msg = d.groups[0].messages
+ assert sys_msg.role == ("developer" if builder is OpenAIResponseAPIMessageBuilder else "system")
+ assert texts(sys_msg) == ["SYSTEM"]
+ assert goal_msg.role == expected_role
+ assert goal_msg.content == [
+ {"text": "# Goal:\n"},
+ {"text": "Do X"},
+ {"image": "data:image/png;base64,AAA"},
+ ]
+
+
+def test_goal_block_default_sys_msg():
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ Goal().apply(llm, d, make_obs())
+ assert texts(d.groups[0].messages[0]) == [SYS_MSG]
+
+
+# --------------------------------------------------------------------------------------
+# Obs block and _format_tabs
+# --------------------------------------------------------------------------------------
+
+
+def test_format_tabs_marks_active_tab():
+ out = _format_tabs(make_obs())
+ assert out.startswith("Currently open tabs:\n")
+ assert "Tab 0:\n Title: A\n URL: http://a.test\n" in out
+ assert "Tab 1 (active tab):\n Title: B\n URL: http://b.test\n" in out
+ assert "Tab 0 (active tab)" not in out
+
+
+def test_format_tabs_no_tabs():
+ obs = make_obs(open_pages_urls=[], open_pages_titles=[])
+ assert _format_tabs(obs) == "Currently open tabs:"
+
+
+def test_obs_block_all_features_without_tool_calls():
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ block = Obs(use_last_error=True, use_screenshot=True, use_axtree=True, use_dom=True)
+ block.use_tabs = True
+ obs = make_obs(last_action_error="Boom")
+ msg = block.apply(llm, d, obs, last_llm_output=LLMOutput())
+ assert d.groups[0].messages == [msg]
+ assert msg.role == "user"
+ t = texts(msg)
+ assert t[0] == "Last action error:\nBoom"
+ assert t[1] == f"AXTree:\n{AXTREE_NOTE}\n[a1] button 'Go'"
+ assert t[2] == "DOM:\n"
+ assert t[3].startswith("Currently open tabs:")
+ images = [c["image"] for c in msg.content if "image" in c]
+ assert len(images) == 1 and images[0].startswith("data:image/png;base64,")
+
+
+def test_obs_block_skips_empty_error_and_disabled_parts():
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ block = Obs(use_last_error=True, use_screenshot=False, use_axtree=False)
+ msg = block.apply(llm, d, make_obs(last_action_error=""), last_llm_output=LLMOutput())
+ assert msg.content == []
+
+
+def test_obs_block_ignores_error_when_disabled():
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ block = Obs(use_last_error=False, use_screenshot=False)
+ msg = block.apply(llm, d, make_obs(last_action_error="err"), last_llm_output=LLMOutput())
+ assert msg.content == []
+
+
+def test_obs_block_uses_som_screenshot(monkeypatch):
+ seen = []
+ monkeypatch.setattr(tua, "image_to_png_base64_url", lambda img: seen.append(img) or "IMG")
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ som = np.ones((2, 2, 3), dtype=np.uint8)
+ msg = Obs(use_som=True).apply(llm, d, make_obs(screenshot_som=som), last_llm_output=LLMOutput())
+ assert seen[0] is som
+ assert msg.content == [{"image": "IMG"}]
+
+
+def test_obs_block_responds_to_previous_tool_calls():
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ previous = llm_output(tool_names=("click", "type"))
+ msg = Obs(use_screenshot=False).apply(llm, d, make_obs(), last_llm_output=previous)
+ assert len(d.groups[0].messages) == 2
+ obs_msg, tool_msg = d.groups[0].messages
+ assert obs_msg is msg
+ assert tool_msg.role == "tool"
+ assert isinstance(tool_msg, OpenAIResponseAPIMessageBuilder)
+ assert tool_msg.responded_tool_calls is previous.tool_calls
+ assert [c.tool_response for c in previous.tool_calls] == [{"text": "See Observation"}] * 2
+
+
+# --------------------------------------------------------------------------------------
+# GeneralHints and Summarizer
+# --------------------------------------------------------------------------------------
+
+
+def test_general_hints_disabled_adds_nothing():
+ d = StructuredDiscussion()
+ d.new_group()
+ assert GeneralHints(use_hints=False).apply(fake_llm(), d) is None
+ assert d.groups[0].messages == []
+
+
+def test_general_hints_enabled_adds_user_message():
+ d = StructuredDiscussion()
+ d.new_group()
+ GeneralHints(use_hints=True).apply(fake_llm(), d)
+ (msg,) = d.groups[0].messages
+ assert msg.role == "user"
+ assert "ControlOrMeta" in texts(msg)[0]
+
+
+def test_summarizer_disabled_is_noop():
+ llm = fake_llm()
+ d = StructuredDiscussion()
+ d.new_group()
+ s = Summarizer(do_summary=False)
+ assert s.apply(llm, d) is None
+ assert s.apply_init(llm, d) is None
+ assert d.groups[0].messages == []
+ assert llm.payloads == []
+
+
+@pytest.mark.parametrize(
+ "high_details, expected",
+ [(True, "# Summarizer instructions:"), (False, "give a semantic description")],
+)
+def test_summarizer_apply_init_instructions(high_details, expected):
+ d = StructuredDiscussion()
+ d.new_group()
+ Summarizer(do_summary=True, high_details=high_details).apply_init(fake_llm(), d)
+ (msg,) = d.groups[0].messages
+ assert msg.role == "developer"
+ assert expected in texts(msg)[0]
+
+
+def test_summarizer_apply_calls_llm_and_sets_summary():
+ llm = fake_llm(responses=[LLMOutput(think="the summary")])
+ d = StructuredDiscussion()
+ d.new_group()
+ d.append(llm.msg.user().add_text("obs"))
+ summary = Summarizer(do_summary=True).apply(llm, d)
+
+ (call,) = llm.payloads
+ assert [texts(m) for m in call.messages] == [["obs"], ["Summarize\n"]]
+ assert call.payload.tools is None
+ assert summary.role == "assistant"
+ assert texts(summary) == ["the summary"]
+ assert d.groups[-1].messages[-1] is summary
+ assert d.get_last_summary() is summary
+
+
+# --------------------------------------------------------------------------------------
+# TaskHint
+# --------------------------------------------------------------------------------------
+
+
+def test_task_hint_relative_path_resolves_next_to_module():
+ block = TaskHint().make() # ships with hint_db.csv next to the module
+ assert {"task_name", "semantic_keys", "hint"} <= set(block.hint_db.columns)
+ assert len(block.hint_db) > 0
+
+
+def test_task_hint_direct_uses_fnmatch_on_task_name(hint_csv):
+ block = TaskHint(hint_db_rel_path=str(hint_csv)).make()
+ assert block.choose_hints_direct("miniwob.book-flight") == [
+ "select from the dropdown list",
+ "activate the field first",
+ ]
+ # the runtime task name is used as the glob pattern
+ assert block.choose_hints_direct("miniwob.*") == [
+ "select from the dropdown list",
+ "activate the field first",
+ "select from the dropdown list",
+ ]
+ assert block.choose_hints_direct("nope") == []
+
+
+def _goal_discussion(builder=OpenAIResponseAPIMessageBuilder, goal="Book flight"):
+ llm = fake_llm(builder)
+ d = StructuredDiscussion()
+ d.new_group("goal")
+ Goal().apply(llm, d, make_obs(goal_object=[{"type": "text", "text": goal}]))
+ return llm, d
+
+
+def test_task_hint_disabled_returns_empty(hint_csv):
+ llm, d = _goal_discussion()
+ block = TaskHint(use_task_hint=False, hint_db_rel_path=str(hint_csv)).make()
+ assert block.apply(llm, d, "miniwob.book-flight") == {}
+ assert len(d.groups[0].messages) == 2
+
+
+def test_task_hint_apply_formats_and_strips_hints(hint_csv):
+ llm, d = _goal_discussion()
+ block = TaskHint(hint_db_rel_path=str(hint_csv)).make()
+ block.apply(llm, d, "workarena.order")
+ hint_msg = d.groups[0].messages[-1]
+ assert hint_msg.role == "user"
+ assert texts(hint_msg) == [
+ "# Hints:\nHere are some hints for the task you are working on:\n- use the form"
+ ]
+
+
+def test_task_hint_apply_no_match_adds_nothing(hint_csv):
+ llm, d = _goal_discussion()
+ block = TaskHint(hint_db_rel_path=str(hint_csv)).make()
+ block.apply(llm, d, "unknown.task")
+ assert len(d.groups[0].messages) == 2
+
+
+def test_task_hint_apply_skips_blank_hints(hint_csv):
+ llm, d = _goal_discussion()
+ block = TaskHint(hint_db_rel_path=str(hint_csv)).make()
+ with mock.patch.object(block, "choose_hints", return_value=[" ", "real"]) as ch:
+ block.apply(llm, d, "t")
+ ch.assert_called_once_with(llm, "t", "# Goal:\n\nBook flight")
+ assert texts(d.groups[0].messages[-1])[0].endswith(":\n- real")
+
+
+def test_task_hint_choose_hints_dispatch(hint_csv):
+ block = TaskHint(hint_db_rel_path=str(hint_csv)).make()
+ llm = fake_llm()
+ with (
+ mock.patch.object(block, "choose_hints_llm", return_value=["L"]) as m_llm,
+ mock.patch.object(block, "choose_hints_emb", return_value=["E"]) as m_emb,
+ ):
+ block.hint_retrieval_mode = "llm"
+ assert block.choose_hints(llm, "t", "g") == ["L"]
+ m_llm.assert_called_once_with(llm, "g")
+ block.hint_retrieval_mode = "emb"
+ assert block.choose_hints(llm, "t", "g") == ["E"]
+ m_emb.assert_called_once_with("g")
+ block.hint_retrieval_mode = "direct"
+ assert block.choose_hints(llm, "workarena.order", "g") == [" use the form "]
+ block.hint_retrieval_mode = "bogus"
+ with pytest.raises(ValueError, match="Unknown hint retrieval mode: bogus"):
+ block.choose_hints(llm, "t", "g")
+
+
+def test_task_hint_llm_mode_selects_topic_and_dedupes(hint_csv):
+ block = TaskHint(hint_db_rel_path=str(hint_csv), hint_retrieval_mode="llm").make()
+ llm = fake_llm(responses=[LLMOutput(think="0")])
+ hints = block.choose_hints_llm(llm, "Book a flight")
+ # topic 0 == "dropdown" (first seen); its duplicate hint is collapsed
+ assert hints == ["select from the dropdown list"]
+ (call,) = llm.payloads
+ prompt = texts(call.messages[0])[0]
+ assert "Book a flight" in prompt
+ assert "0. dropdown\n1. forms" in prompt
+
+
+def test_task_hint_llm_mode_second_topic(hint_csv):
+ block = TaskHint(hint_db_rel_path=str(hint_csv), hint_retrieval_mode="llm").make()
+ llm = fake_llm(responses=[LLMOutput(think="1")])
+ assert block.choose_hints_llm(llm, "g") == ["activate the field first", " use the form "]
+
+
+@pytest.mark.parametrize("answer", ["-1", "2", "99", "not json"])
+def test_task_hint_llm_mode_bad_answers_return_no_hints(hint_csv, answer):
+ block = TaskHint(hint_db_rel_path=str(hint_csv), hint_retrieval_mode="llm").make()
+ llm = fake_llm(responses=[LLMOutput(think=answer)])
+ assert block.choose_hints_llm(llm, "g") == []
+
+
+@pytest.mark.xfail(
+ strict=True,
+ reason="BUG: non-integer JSON answer (e.g. '\"1\"') raises TypeError instead of no hints",
+)
+def test_task_hint_llm_mode_quoted_number_returns_no_hints(hint_csv):
+ block = TaskHint(hint_db_rel_path=str(hint_csv), hint_retrieval_mode="llm").make()
+ llm = fake_llm(responses=[LLMOutput(think='"1"')])
+ assert block.choose_hints_llm(llm, "g") == []
+
+
+def _emb_setup(tmp_path, monkeypatch):
+ rows = [
+ ("t1", "k1", "hint one"),
+ ("t2", "k2", "hint two"),
+ ("t3", "k1", "hint one"), # duplicate hint, dropped by encode_hints
+ ("t4", "k3", "hint three"),
+ ]
+ csv = write_hint_db(tmp_path / "db.csv", rows)
+ embs = {
+ "k1: hint one": [1.0, 0.0],
+ "k2: hint two": [0.0, 1.0],
+ "k3: hint three": [0.5, 0.5],
+ }
+ np.save(f"{csv}.embs.npy", embs, allow_pickle=True)
+ monkeypatch.chdir(tmp_path)
+ return csv
+
+
+def test_task_hint_emb_init_loads_embeddings(tmp_path, monkeypatch):
+ csv = _emb_setup(tmp_path, monkeypatch)
+ block = TaskHint(hint_db_rel_path=str(csv), hint_retrieval_mode="emb").make()
+ assert block.uniq_hints["hint"].tolist() == ["hint one", "hint two", "hint three"]
+ np.testing.assert_array_equal(block.hint_embeddings, [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]])
+
+
+def test_task_hint_emb_missing_embeddings_file_asserts(tmp_path, hint_csv):
+ with pytest.raises(AssertionError, match="Embedding file not found"):
+ TaskHint(hint_db_rel_path=str(hint_csv), hint_retrieval_mode="emb").make()
+
+
+@pytest.mark.xfail(
+ strict=True,
+ reason="BUG: encode_hints resolves '.embs.npy' against CWD, not the CSV's directory",
+)
+def test_task_hint_emb_relative_path_resolves_next_to_module(tmp_path, monkeypatch):
+ module_dir = tmp_path / "module"
+ module_dir.mkdir()
+ csv = write_hint_db(module_dir / "hint_db.csv", [("t", "k", "h")])
+ np.save(f"{csv}.embs.npy", {"k: h": [1.0]}, allow_pickle=True)
+ monkeypatch.setattr(tua, "__file__", str(module_dir / "tool_use_agent.py"))
+ elsewhere = tmp_path / "elsewhere"
+ elsewhere.mkdir()
+ monkeypatch.chdir(elsewhere)
+ block = TaskHint(hint_db_rel_path="hint_db.csv", hint_retrieval_mode="emb").make()
+ assert block.hint_embeddings.shape == (1, 1)
+
+
+def _response(payload):
+ return SimpleNamespace(json=lambda: payload)
+
+
+def test_task_hint_choose_hints_emb_returns_top_n(tmp_path, monkeypatch):
+ csv = _emb_setup(tmp_path, monkeypatch)
+ block = TaskHint(hint_db_rel_path=str(csv), hint_retrieval_mode="emb", top_n=2).make()
+ posts = []
+
+ def fake_post(url, json, timeout):
+ posts.append((url, json, timeout))
+ if url.endswith("/encode"):
+ return _response({"embeddings": [[0.1, 0.9]]})
+ return _response({"similarities": [[0.2, 0.9, 0.5]]})
+
+ monkeypatch.setattr(tua.requests, "post", fake_post)
+ hints = block.choose_hints_emb("my goal")
+ # top-2 by ascending argsort: index 2 (0.5) then index 1 (0.9)
+ assert hints == ["hint three", "hint two"]
+ assert posts[0] == (
+ "http://localhost:5000/encode",
+ {"texts": ["my goal"], "prompt": "task description"},
+ 10,
+ )
+ assert posts[1][0] == "http://localhost:5000/similarity"
+ assert posts[1][1]["texts1"] == [[0.1, 0.9]]
+ assert posts[1][1]["texts2"] == [[1.0, 0.0], [0.0, 1.0], [0.5, 0.5]]
+ assert posts[1][2] == 2
+
+
+def test_task_hint_oai_embed_uses_client():
+ block = TaskHint(use_task_hint=False)
+ client = mock.Mock()
+ client.create.return_value = SimpleNamespace(data=[SimpleNamespace(embedding=[0.1, 0.2])])
+ block._oai_emb = client
+ assert block.oai_embed("hello") == [0.1, 0.2]
+ client.create.assert_called_once_with(input="hello", model="text-embedding-3-small")
+
+
+@pytest.mark.parametrize(
+ "method, args, key, timeout",
+ [
+ ("_encode", (["a"],), "embeddings", 10),
+ ("_similarity", ([[1.0]], [[1.0]]), "similarities", 2),
+ ],
+)
+def test_task_hint_http_helpers_retry_then_succeed(monkeypatch, method, args, key, timeout):
+ block = TaskHint(use_task_hint=False, embedder_server="http://emb.test")
+ outcomes = [requests.exceptions.ConnectionError("down"), requests.exceptions.Timeout("slow")]
+
+ def fake_post(url, json, timeout):
+ if outcomes:
+ raise outcomes.pop(0)
+ return _response({key: [[0.5]]})
+
+ sleeps, uniforms = [], []
+ monkeypatch.setattr(tua.requests, "post", fake_post)
+ monkeypatch.setattr(tua.time, "sleep", sleeps.append)
+ monkeypatch.setattr(tua.random, "uniform", lambda a, b: uniforms.append((a, b)) or 0.25)
+
+ out = getattr(block, method)(*args)
+ np.testing.assert_array_equal(out, [[0.5]])
+ assert sleeps == [0.25, 0.25]
+ assert uniforms == [(1, timeout), (1, timeout)]
+
+
+@pytest.mark.parametrize("method, args", [("_encode", (["a"],)), ("_similarity", ([1], [1]))])
+def test_task_hint_http_helpers_raise_after_max_retries(monkeypatch, method, args):
+ block = TaskHint(use_task_hint=False)
+ calls = []
+
+ def fake_post(url, json, timeout):
+ calls.append(url)
+ raise requests.exceptions.ConnectionError("down")
+
+ sleeps = []
+ monkeypatch.setattr(tua.requests, "post", fake_post)
+ monkeypatch.setattr(tua.time, "sleep", sleeps.append)
+ monkeypatch.setattr(tua.random, "uniform", lambda a, b: 0.0)
+ with pytest.raises(requests.exceptions.ConnectionError):
+ getattr(block, method)(*args, max_retries=3)
+ assert len(calls) == 3
+ assert len(sleeps) == 2 # no sleep after the final attempt
+
+
+# --------------------------------------------------------------------------------------
+# ToolUseAgentArgs
+# --------------------------------------------------------------------------------------
+
+
+def test_agent_args_name_from_model_name():
+ args = ToolUseAgentArgs(model_args=FakeModelArgs(model_name="org/model-x"))
+ assert args.agent_name == "ToolUse-org_model-x"
+
+
+def test_agent_args_without_model_keeps_default_name():
+ args = ToolUseAgentArgs()
+ assert args.agent_name == ToolUseAgentArgs.__dataclass_fields__["agent_name"].default
+
+
+def test_agent_args_prepare_and_close_delegate_to_model_args():
+ model_args = FakeModelArgs()
+ args = ToolUseAgentArgs(model_args=model_args)
+ assert args.prepare() == "prepared"
+ assert args.close() == "closed"
+ assert (model_args.prepared, model_args.closed) == (1, 1)
+
+
+def test_agent_args_make_agent_with_config(hint_csv):
+ cfg = make_config(hint_csv)
+ action_set = FakeActionSet()
+ args = ToolUseAgentArgs(model_args=FakeModelArgs(), config=cfg, action_set=action_set)
+ agent = args.make_agent()
+ assert isinstance(agent, ToolUseAgent)
+ assert agent.config is cfg
+ assert agent.action_set is action_set
+ assert action_set.apis == ["openai"]
+
+
+def test_agent_args_make_agent_defaults_to_default_prompt_config():
+ args = ToolUseAgentArgs(model_args=FakeModelArgs(), action_set=FakeActionSet())
+ agent = args.make_agent()
+ assert args.config is DEFAULT_PROMPT_CONFIG
+ assert agent.config is DEFAULT_PROMPT_CONFIG
+ assert agent.task_hint.use_task_hint is True
+ assert len(agent.task_hint.hint_db) > 0 # bundled hint_db.csv
+ assert agent.discussion.keep_last_n_obs is None
+
+
+@pytest.mark.parametrize("name, expected", [("osworld", True), ("miniwob", False)])
+def test_agent_args_set_benchmark(hint_csv, name, expected):
+ args = ToolUseAgentArgs(model_args=FakeModelArgs(), config=make_config(hint_csv))
+ args.set_benchmark(SimpleNamespace(name=name), demo_mode=False)
+ assert args.config.obs.skip_preprocessing is expected
+
+
+def test_predefined_configs_are_consistent():
+ assert tua.OSWORLD_CLAUDE.config.task_hint.use_task_hint is False
+ assert isinstance(tua.OSWORLD_OAI.action_set, OSWorldActionSet)
+ assert tua.OSWORLD_OAI.config.keep_last_n_obs == 1
+ assert tua.AGENT_CONFIG.agent_name == "ToolUse-claude-3-7-sonnet-20250219"
+ assert tua.OAI_OPENROUTER_AGENT.agent_name == "ToolUse-openai_gpt-4.1"
+ assert DEFAULT_PROMPT_CONFIG.action_subsets == ("coord",)
+
+
+# --------------------------------------------------------------------------------------
+# ToolUseAgent construction and obs_preprocessor
+# --------------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+ "api, builder",
+ [("openai", OpenAIResponseAPIMessageBuilder), ("anthropic", AnthropicAPIMessageBuilder)],
+)
+def test_agent_init_default_action_set_from_bgym(hint_csv, api, builder):
+ model_args = FakeModelArgs(api=api, builder=builder)
+ agent = ToolUseAgent(model_args=model_args, config=make_config(hint_csv))
+ names = {t["name"] for t in agent.tools}
+ assert "mouse_click" in names
+ assert "noop" in names
+ key = "input_schema" if api == "anthropic" else "parameters"
+ assert all(key in t for t in agent.tools)
+ assert agent.llm.msg is builder
+ assert agent.msg_builder is builder
+ assert agent.call_ids == []
+ assert agent.last_response == LLMOutput()
+
+
+def test_agent_init_with_osworld_action_set(hint_csv):
+ agent = ToolUseAgent(
+ model_args=FakeModelArgs(),
+ config=make_config(hint_csv),
+ action_set=OSWorldActionSet("computer_13"),
+ )
+ assert len(agent.tools) > 0
+ assert all("name" in t.get("function", t) for t in agent.tools)
+
+
+def test_agent_init_copies_blocks(hint_csv):
+ cfg = make_config(hint_csv)
+ agent = ToolUseAgent(model_args=FakeModelArgs(), config=cfg, action_set=FakeActionSet())
+ assert agent.obs_block == cfg.obs and agent.obs_block is not cfg.obs
+ assert agent.task_hint is not cfg.task_hint
+
+
+def test_obs_preprocessor_skip_returns_shallow_copy(hint_csv):
+ agent, _, _ = make_agent(hint_csv, [], obs=Obs(skip_preprocessing=True))
+ obs = {"page": object(), "x": 1}
+ out = agent.obs_preprocessor(obs)
+ assert out == obs and out is not obs
+
+
+def test_obs_preprocessor_page_extracts_screenshot(hint_csv, monkeypatch):
+ agent, _, _ = make_agent(hint_csv, [], obs=Obs(use_axtree=True, use_dom=True))
+ page = object()
+ shot = np.zeros((1, 1, 3), dtype=np.uint8)
+ extract = mock.Mock(return_value=shot)
+ flatten = mock.Mock()
+ monkeypatch.setattr(tua, "extract_screenshot", extract)
+ monkeypatch.setattr(tua, "flatten_axtree_to_str", flatten)
+ obs = {"page": page, "other": 1}
+ out = agent.obs_preprocessor(obs)
+ extract.assert_called_once_with(page)
+ flatten.assert_not_called() # page branch skips text extraction
+ assert out == {"screenshot": shot, "other": 1}
+ assert "page" in obs # input not mutated
+
+
+def test_obs_preprocessor_builds_text_and_som(hint_csv, monkeypatch):
+ agent, _, _ = make_agent(
+ hint_csv,
+ [],
+ obs=Obs(use_axtree=True, use_dom=True, use_som=True, use_zoomed_webpage=True),
+ )
+ calls = {}
+ monkeypatch.setattr(
+ tua,
+ "flatten_dom_to_str",
+ lambda dom, extra_properties: calls.setdefault("dom", (dom, extra_properties)) and "DOM",
+ )
+ monkeypatch.setattr(tua, "prune_html", lambda s: f"pruned({s})")
+ monkeypatch.setattr(
+ tua,
+ "flatten_axtree_to_str",
+ lambda ax, extra_properties: calls.setdefault("ax", (ax, extra_properties)) and "AX",
+ )
+ monkeypatch.setattr(
+ tua,
+ "overlay_som",
+ lambda shot, extra_properties: calls.setdefault("som", (shot, extra_properties)) and "SOM",
+ )
+ extra = {"a1": {"visibility": 1}}
+ obs = {
+ "dom_object": "dom",
+ "axtree_object": "ax",
+ "screenshot": "shot",
+ "extra_element_properties": extra,
+ }
+ out = agent.obs_preprocessor(obs)
+ assert out["dom_txt"] == "DOM"
+ assert out["pruned_html"] == "pruned(DOM)"
+ assert out["axtree_txt"] == "AX"
+ assert out["screenshot_som"] == "SOM"
+ assert calls == {"dom": ("dom", extra), "ax": ("ax", extra), "som": ("shot", extra)}
+ assert "axtree_txt" not in obs
+
+
+def test_obs_preprocessor_with_real_browsergym_helpers(hint_csv):
+ agent, _, _ = make_agent(hint_csv, [], obs=Obs(use_axtree=True))
+ axtree = {
+ "nodes": [
+ {
+ "nodeId": "1",
+ "role": {"value": "RootWebArea"},
+ "name": {"value": "Page"},
+ "childIds": ["2"],
+ "browsergym_id": "r1",
+ },
+ {
+ "nodeId": "2",
+ "role": {"value": "button"},
+ "name": {"value": "Go"},
+ "childIds": [],
+ "browsergym_id": "a1",
+ },
+ ]
+ }
+ out = agent.obs_preprocessor({"axtree_object": axtree, "extra_element_properties": {}})
+ assert "button 'Go'" in out["axtree_txt"]
+ assert "[a1]" in out["axtree_txt"]
+
+
+# --------------------------------------------------------------------------------------
+# ToolUseAgent.get_action loop
+# --------------------------------------------------------------------------------------
+
+
+def test_get_action_first_step_builds_goal_and_obs(hint_csv):
+ response = llm_output(action="click('a1')", think="I will click")
+ agent, llm, action_set = make_agent(hint_csv, [response])
+ action, info = agent.get_action(make_obs())
+
+ assert action == "click('a1')"
+ assert info.think == "I will click"
+ assert llm.reset_count == 1
+ assert agent.last_response is response
+ assert agent._responses == [response]
+
+ (call,) = llm.payloads
+ p = call.payload
+ assert p.tools == action_set.to_tool_description()
+ assert p.tool_choice == "any"
+ assert p.cache_tool_definition is True
+ assert p.cache_complete_prompt is False
+ assert p.use_cache_breakpoints is True
+ roles = [m.role for m in call.messages]
+ assert roles == ["developer", "developer", "user"]
+ assert texts(call.messages[0])[0].endswith("You can only take one action at a time.")
+ assert texts(call.messages[1]) == ["# Goal:\n", "Book a flight to Paris"]
+
+ # chat_messages for xray: tool description first, tool-call message last
+ chat = info.chat_messages
+ assert chat[0].role == "tool_description"
+ assert json.loads(texts(chat[0])[0]) == p.tools
+ assert chat[-1].role == "tool"
+ assert chat[-1].responded_tool_calls is response.tool_calls
+ assert chat[1:-1] == call.messages
+
+ # stats from model + cost tracker merged
+ assert info.stats["n_api_calls"] == 1
+ assert {"input_tokens", "output_tokens", "cost"} <= set(info.stats)
+
+ # groups: goal, step-0 obs, new empty group for the next step
+ assert [g.name for g in agent.discussion.groups] == ["goal", "group_1", "group_2"]
+ assert agent.discussion.groups[-1].messages == []
+
+
+def test_get_action_multiaction_system_prompt(hint_csv):
+ agent, llm, _ = make_agent(hint_csv, [llm_output()], multiaction=True)
+ agent.get_action(make_obs())
+ first = texts(llm.payloads[0].messages[0])[0]
+ assert first.endswith("You can take multiple actions in a single step, if needed.")
+
+
+def test_get_action_second_step_reuses_goal_and_answers_tool_calls(hint_csv):
+ r1 = llm_output(action="a1", think="t1", tool_names=("click",))
+ r2 = llm_output(action="a2", think="t2", tool_names=("type",))
+ agent, llm, _ = make_agent(hint_csv, [r1, r2])
+ agent.get_action(make_obs())
+ action, info = agent.get_action(make_obs(last_action_error="oops"))
+
+ assert action == "a2"
+ assert info.think == "t2"
+ assert llm.reset_count == 2
+ second = llm.payloads[1].messages
+ assert [m.role for m in second] == ["developer", "developer", "user", "user", "tool"]
+ assert texts(second[3])[0] == "Last action error:\noops"
+ assert second[4].responded_tool_calls is r1.tool_calls
+ assert r1.tool_calls.tool_calls[0].tool_response == {"text": "See Observation"}
+ assert agent._responses == [r1, r2]
+ # goal not re-added
+ assert sum(1 for g in agent.discussion.groups if g.name == "goal") == 1
+
+
+def test_get_action_with_summary_hints_and_task_hint(hint_csv):
+ responses = [LLMOutput(think="SUMMARY-1"), llm_output(action="act", think="reason")]
+ agent, llm, _ = make_agent(
+ hint_csv,
+ responses,
+ builder=AnthropicAPIMessageBuilder,
+ summarizer=Summarizer(do_summary=True, high_details=False),
+ general_hints=GeneralHints(use_hints=True),
+ task_hint=TaskHint(use_task_hint=True, hint_db_rel_path=str(hint_csv)),
+ )
+ action, info = agent.get_action(make_obs())
+
+ assert action == "act"
+ assert info.think == "SUMMARY-1\nreason"
+ summary_call, action_call = llm.payloads
+ assert summary_call.payload.tools is None
+ assert texts(summary_call.messages[-1]) == ["Summarize\n"]
+
+ msgs = action_call.messages
+ assert [m.role for m in msgs] == [
+ "system", # sys msg
+ "system", # goal
+ "system", # summarizer instructions
+ "user", # general hints
+ "user", # task hints
+ "user", # obs
+ "user", # "Summarize"
+ "assistant", # summary
+ ]
+ assert "- select from the dropdown list" in texts(msgs[4])[0]
+ assert "- activate the field first" in texts(msgs[4])[0]
+ assert texts(msgs[-1]) == ["SUMMARY-1"]
+ assert agent.discussion.groups[1].summary is msgs[-1]
+
+
+def test_get_action_keep_last_n_obs_summarizes_old_steps(hint_csv):
+ responses = [
+ LLMOutput(think="S1"),
+ llm_output(action="a1", think="t1"),
+ LLMOutput(think="S2"),
+ llm_output(action="a2", think="t2"),
+ ]
+ agent, llm, _ = make_agent(
+ hint_csv,
+ responses,
+ builder=AnthropicAPIMessageBuilder,
+ summarizer=Summarizer(do_summary=True),
+ keep_last_n_obs=1,
+ )
+ agent.get_action(make_obs(axtree_txt="FIRST"))
+ _, info = agent.get_action(make_obs(axtree_txt="SECOND"))
+ assert info.think == "S2\nt2"
+
+ final_msgs = llm.payloads[-1].messages
+ all_text = "\n".join(t for m in final_msgs for t in texts(m))
+ assert "SECOND" in all_text
+ assert "FIRST" not in all_text # step 1 replaced by its summary
+ assert "S1" in all_text
+ assert sum(getattr(m, "_cache_breakpoint", False) for m in final_msgs) == 1
+
+
+def test_get_action_nested_cost_tracker_accumulates(hint_csv):
+ from agentlab.llm.tracking import set_tracker
+
+ agent, _, _ = make_agent(hint_csv, [llm_output()])
+ with set_tracker() as outer:
+ _, info = agent.get_action(make_obs())
+ assert info.stats["cost"] == 0
+ assert outer.cost == 0
+ assert getattr(TRACKER, "instance", None) is None