diff --git a/tests/analyze/test_agent_xray_helpers.py b/tests/analyze/test_agent_xray_helpers.py new file mode 100644 index 000000000..1af23abfd --- /dev/null +++ b/tests/analyze/test_agent_xray_helpers.py @@ -0,0 +1,867 @@ +import base64 +import json +import math +import os +import re +from types import SimpleNamespace +from unittest.mock import MagicMock + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +import pandas as pd # noqa: E402 +import pytest # noqa: E402 +from openai.types.responses import ResponseFunctionToolCall # noqa: E402 +from PIL import Image # noqa: E402 + +from agentlab.analyze import agent_xray as ax # noqa: E402 +from agentlab.experiments.loop import ExpResult, StepInfo, StepTimestamps # noqa: E402 +from agentlab.llm.llm_utils import Discussion # noqa: E402 +from agentlab.llm.response_api import MessageBuilder, ToolCall, ToolCalls # noqa: E402 + +GR_UPDATE = {"__type__": "update"} + + +@pytest.fixture(autouse=True) +def fresh_info(monkeypatch): + """Give every test its own module-global `info`; monkeypatch restores the original.""" + new_info = ax.Info() + monkeypatch.setattr(ax, "info", new_info) + yield new_info + plt.close("all") + + +def _img_array(color, size=(8, 6)): + arr = np.zeros((size[1], size[0], 3), dtype=np.uint8) + arr[:, :] = color + return arr + + +def _make_exp_result(tmp_path, steps, summary=None, logs=None, name="exp"): + exp_dir = tmp_path / "study" / name + exp_dir.mkdir(parents=True) + for s in steps: + s.save_step_info(exp_dir, save_som=True) + if summary is not None: + (exp_dir / "summary_info.json").write_text(json.dumps(summary)) + if logs is not None: + (exp_dir / "experiment.log").write_text(logs) + exp = ExpResult(exp_dir) + exp._exp_args = SimpleNamespace(env_args=SimpleNamespace(task_name="task.a", task_seed=3)) + return exp + + +GOAL = [{"type": "text", "text": "Do it"}] + + +def _step(i, action=None, obs=None, agent_info=None, screens=True, goal=True, **kw): + obs = dict(obs or {}) + if goal: + obs.setdefault("goal_object", GOAL) + if screens: + obs["screenshot"] = _img_array((20 * i, 0, 0)) + obs["screenshot_som"] = _img_array((0, 20 * i, 0)) + return StepInfo(step=i, obs=obs, action=action, agent_info=agent_info or {}, **kw) + + +@pytest.fixture +def episode(tmp_path, fresh_info): + steps = [ + _step(0, task_info={"k": 1}), + _step( + 1, + action="click('a1')", + obs={ + "axtree_txt": "AX", + "dom_txt": "
", + "pruned_html": "", + "extra_element_properties": {"a1": {"bbox": (1, 1, 3, 3)}}, + }, + agent_info={ + "think": "thinking", + "chat_messages": [ + {"role": "system", "content": [{"type": "text", "text": "sys"}]}, + {"role": "user", "content": [{"type": "text", "text": "user prompt"}]}, + ], + "markdown_page": "# md", + "html_page": "hi", + }, + stats={"n_tokens": 5, "cost": 0.1}, + ), + _step(2, action="noop()", obs={"last_action_error": "boom"}, terminated=True), + ] + exp = _make_exp_result( + tmp_path, + steps, + summary={"cum_reward": 1.0, "err_msg": None, "stack_trace": "Traceback..."}, + logs="log line", + ) + fresh_info.exp_result = exp + fresh_info.step = 1 + return exp + + +# --------------------------------------------------------------------------- tables + + +def test_display_table_wraps_dotted_names_without_mutating(): + df = pd.DataFrame({"agent.name": [1], "b": [2]}).set_index("agent.name") + df2 = pd.DataFrame({"env.task_name": [1]}, index=pd.Index([0], name="x.y")) + out = ax.display_table(df2) + assert list(out.columns) == ["env.\ntask_name"] + assert list(out.index.names) == ["x.\ny"] + assert list(df2.columns) == ["env.task_name"] + assert list(ax.display_table(df).index.names) == ["agent.\nname"] + + +def test_remove_args_from_col_in_place(): + df = pd.DataFrame({"agent_args.x": [1], "env_args.task_name": [2]}) + df.index.name = "agent_args.name" + out = ax.remove_args_from_col(df) + assert out is df + assert list(df.columns) == ["agent.x", "env.task_name"] + assert list(df.index.names) == ["agent.name"] + + +@pytest.mark.parametrize( + "cols, expected", + [([], []), (["a"], ["a"]), (["a.b.c", "d_args"], ["a.\nb.\nc", "d_args"])], +) +def test_clean_column_names(cols, expected): + assert ax.clean_column_names(cols) == expected + + +def test_click_mapper_maps_pixels_to_steps(): + fig, ax_ = plt.subplots() + ax_.set_xlim(0, 10) + fig.canvas.draw() + mapper = ax.ClickMapper(ax_, step_times=[2.0, 5.0, 8.0]) + for t, expected_step in [(1.0, 0), (3.0, 1), (6.0, 2), (9.0, 3)]: + x_pix, _ = ax_.transData.transform((t, 0)) + assert mapper.to_time(x_pix) == pytest.approx(t) + assert mapper.to_step(x_pix) == expected_step + + +def test_dataclass_defaults(): + assert ax.EpisodeId() == ax.EpisodeId(None, None, None, None) + assert ax.StepId().step is None + info = ax.Info() + assert info.active_tab == "Screenshot" + assert info.exp_result is None + + +# --------------------------------------------------------------------------- Info + + +def _result_df(tmp_path): + return pd.DataFrame( + { + ax.TASK_NAME_KEY: ["t1", "t1", "t2"], + ax.TASK_SEED_KEY: [0, 1, 0], + "agent.agent_name": ["A", "A", "B"], + "cum_reward": [1.0, 0.0, 0.5], + "err_msg": [None, "err", None], + "n_steps": [3, 4, 5], + "exp_dir": [tmp_path / "e0", tmp_path / "e1", tmp_path / "e2"], + } + ) + + +class TestInfo: + def test_get_agent_id(self): + info = ax.Info(agent_id_keys=["agent.agent_name", "x"]) + row = pd.Series({"agent.agent_name": "A", "x": 2, "y": 3}) + assert info.get_agent_id(row) == [("agent.agent_name", "A"), ("x", 2)] + + def test_filter_agent_id_preserves_row_index(self, tmp_path): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.\nagent_name", "A")]) + assert list(info.agent_df.index) == ["t1", "t1"] + assert list(info.agent_df["_row_index"]) == [0, 1] + info.filter_agent_id([("agent.agent_name", "B")]) + assert list(info.agent_df["_row_index"]) == [2] + + def test_update_exp_result(self, tmp_path): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.agent_name", "A")]) + info.step = 5 + info.update_exp_result(ax.EpisodeId("A", "t1", 1, row_index=1)) + assert isinstance(info.exp_result, ExpResult) + assert info.exp_result.exp_dir == tmp_path / "e1" + assert info.step == 0 + + def test_update_exp_result_unknown_row(self, tmp_path): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.agent_name", "A")]) + with pytest.raises(ValueError, match="_row_index: 2"): + info.update_exp_result(ax.EpisodeId("A", "t2", 0, row_index=2)) + assert info.exp_result is None + + def test_update_exp_result_duplicate_rows_warns(self, tmp_path, monkeypatch): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.agent_name", "A")]) + info.agent_df["_row_index"] = 0 + warn = MagicMock() + monkeypatch.setattr(ax, "warning", warn) + info.update_exp_result(ax.EpisodeId("A", "t1", 0, row_index=0)) + assert "multiple rows" in warn.call_args[0][0] + assert info.exp_result.exp_dir == tmp_path / "e0" + + @pytest.mark.xfail( + strict=True, + reason="BUG: update_exp_result does not return after the result_df-is-None guard", + ) + def test_update_exp_result_without_results_clears_selection(self): + info = ax.Info() + info.update_exp_result(ax.EpisodeId()) + assert info.exp_result is None + + +# --------------------------------------------------------------------------- markdown + + +class TestDictToMarkdown: + def test_tool_calls_render_empty(self): + assert ax.dict_to_markdown(ToolCalls()) == "" + + def test_non_dict_returns_repr_and_warns(self, monkeypatch): + warn = MagicMock() + monkeypatch.setattr(ax, "warning", warn) + assert ax.dict_to_markdown([1, "a"]) == "[1, 'a']" + warn.assert_called_once() + + def test_empty(self): + assert ax.dict_to_markdown({}) == "No Data" + + def test_nested(self): + d = {"a": 1, "b": {"c": "x"}, "l": [{"k": 2}, "s"], "e": {}} + assert ax.dict_to_markdown(d) == ( + "- **a**: 1\n" + "### b\n- **c**: x\n\n" + "### l\n#### Item 0\n- **k**: 2\n\n- s\n" + "### e\nNo Data\n" + ) + + +class TestDictMsgToMarkdown: + def test_without_role_falls_back(self): + assert ax.dict_msg_to_markdown({"x": 1}) == "- **x**: 1\n" + + def test_content_types(self): + class PydanticLike: + def dict(self): + return {"type": "text", "text": "from dict()"} + + msg = { + "role": "assistant", + "content": [ + {"type": "image", "image": "data:img"}, + {"type": "text", "text": "hello"}, + {"type": "tool_use", "name": "click", "input": {"bid": "a"}, "id": "t1"}, + {"type": "weird", "v": 1}, + PydanticLike(), + ], + } + out = ax.dict_msg_to_markdown(msg) + assert out.startswith("### Assistant\n") + assert "" in out + assert "\n```\nhello\n```\n" in out + assert "Tool Call: click `{'bid': 'a'}` (call_id: t1)" in out + assert "{'type': 'weird', 'v': 1}" in out + assert "from dict()" in out + + @pytest.mark.xfail( + strict=True, + reason="BUG: dict messages with plain-string content crash (iterates characters)", + ) + def test_string_content(self): + out = ax.dict_msg_to_markdown({"role": "user", "content": "hello"}) + assert "hello" in out + + +def test_format_tool_call(): + assert ax._format_tool_call("f", '{"a": 1}', "c9") == 'Tool Call: f `{"a": 1}` (call_id: c9)' + + +class TestFormatChatMessage: + def test_langchain_message(self, monkeypatch): + class FakeLCMessage: + def __init__(self, content): + self.content = content + + monkeypatch.setattr(ax, "BaseMessage", FakeLCMessage) + assert ax.format_chat_message(FakeLCMessage("lc content")) == "lc content" + + def test_message_builder(self): + mb = MessageBuilder.assistant().add_text("yo") + assert ax.format_chat_message(mb) == "### Assistant\n\n```\nyo\n```\n" + + def test_message_builder_with_tool_calls(self): + tc = ToolCalls([ToolCall(name="click", arguments={"bid": "a1"}).response_text("ok")]) + md = ax.format_chat_message(MessageBuilder.add_responded_tool_calls(tc)) + assert "**Tool Call 1**: click(bid='a1')" in md + assert "ok" in md + + def test_dict(self): + out = ax.format_chat_message({"role": "user", "content": [{"type": "text", "text": "q"}]}) + assert out == "### User\n\n```\nq\n```\n" + + def test_response_function_tool_call(self): + call = ResponseFunctionToolCall( + name="fill", arguments='{"v": 1}', call_id="c1", type="function_call" + ) + assert ax.format_chat_message(call) == ( + '### Tool Use\n```\nTool Call: fill `{"v": 1}` (call_id: c1)\n```\n' + ) + + def test_other(self): + assert ax.format_chat_message(12) == "12" + assert ax.format_chat_message("No Chat Messages") == "No Chat Messages" + + +def test_page_to_iframe_roundtrip(): + page = "