diff --git a/tests/analyze/test_agent_xray_helpers.py b/tests/analyze/test_agent_xray_helpers.py new file mode 100644 index 000000000..1af23abfd --- /dev/null +++ b/tests/analyze/test_agent_xray_helpers.py @@ -0,0 +1,867 @@ +import base64 +import json +import math +import os +import re +from types import SimpleNamespace +from unittest.mock import MagicMock + +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 +import pandas as pd # noqa: E402 +import pytest # noqa: E402 +from openai.types.responses import ResponseFunctionToolCall # noqa: E402 +from PIL import Image # noqa: E402 + +from agentlab.analyze import agent_xray as ax # noqa: E402 +from agentlab.experiments.loop import ExpResult, StepInfo, StepTimestamps # noqa: E402 +from agentlab.llm.llm_utils import Discussion # noqa: E402 +from agentlab.llm.response_api import MessageBuilder, ToolCall, ToolCalls # noqa: E402 + +GR_UPDATE = {"__type__": "update"} + + +@pytest.fixture(autouse=True) +def fresh_info(monkeypatch): + """Give every test its own module-global `info`; monkeypatch restores the original.""" + new_info = ax.Info() + monkeypatch.setattr(ax, "info", new_info) + yield new_info + plt.close("all") + + +def _img_array(color, size=(8, 6)): + arr = np.zeros((size[1], size[0], 3), dtype=np.uint8) + arr[:, :] = color + return arr + + +def _make_exp_result(tmp_path, steps, summary=None, logs=None, name="exp"): + exp_dir = tmp_path / "study" / name + exp_dir.mkdir(parents=True) + for s in steps: + s.save_step_info(exp_dir, save_som=True) + if summary is not None: + (exp_dir / "summary_info.json").write_text(json.dumps(summary)) + if logs is not None: + (exp_dir / "experiment.log").write_text(logs) + exp = ExpResult(exp_dir) + exp._exp_args = SimpleNamespace(env_args=SimpleNamespace(task_name="task.a", task_seed=3)) + return exp + + +GOAL = [{"type": "text", "text": "Do it"}] + + +def _step(i, action=None, obs=None, agent_info=None, screens=True, goal=True, **kw): + obs = dict(obs or {}) + if goal: + obs.setdefault("goal_object", GOAL) + if screens: + obs["screenshot"] = _img_array((20 * i, 0, 0)) + obs["screenshot_som"] = _img_array((0, 20 * i, 0)) + return StepInfo(step=i, obs=obs, action=action, agent_info=agent_info or {}, **kw) + + +@pytest.fixture +def episode(tmp_path, fresh_info): + steps = [ + _step(0, task_info={"k": 1}), + _step( + 1, + action="click('a1')", + obs={ + "axtree_txt": "AX", + "dom_txt": "", + "pruned_html": "

", + "extra_element_properties": {"a1": {"bbox": (1, 1, 3, 3)}}, + }, + agent_info={ + "think": "thinking", + "chat_messages": [ + {"role": "system", "content": [{"type": "text", "text": "sys"}]}, + {"role": "user", "content": [{"type": "text", "text": "user prompt"}]}, + ], + "markdown_page": "# md", + "html_page": "hi", + }, + stats={"n_tokens": 5, "cost": 0.1}, + ), + _step(2, action="noop()", obs={"last_action_error": "boom"}, terminated=True), + ] + exp = _make_exp_result( + tmp_path, + steps, + summary={"cum_reward": 1.0, "err_msg": None, "stack_trace": "Traceback..."}, + logs="log line", + ) + fresh_info.exp_result = exp + fresh_info.step = 1 + return exp + + +# --------------------------------------------------------------------------- tables + + +def test_display_table_wraps_dotted_names_without_mutating(): + df = pd.DataFrame({"agent.name": [1], "b": [2]}).set_index("agent.name") + df2 = pd.DataFrame({"env.task_name": [1]}, index=pd.Index([0], name="x.y")) + out = ax.display_table(df2) + assert list(out.columns) == ["env.\ntask_name"] + assert list(out.index.names) == ["x.\ny"] + assert list(df2.columns) == ["env.task_name"] + assert list(ax.display_table(df).index.names) == ["agent.\nname"] + + +def test_remove_args_from_col_in_place(): + df = pd.DataFrame({"agent_args.x": [1], "env_args.task_name": [2]}) + df.index.name = "agent_args.name" + out = ax.remove_args_from_col(df) + assert out is df + assert list(df.columns) == ["agent.x", "env.task_name"] + assert list(df.index.names) == ["agent.name"] + + +@pytest.mark.parametrize( + "cols, expected", + [([], []), (["a"], ["a"]), (["a.b.c", "d_args"], ["a.\nb.\nc", "d_args"])], +) +def test_clean_column_names(cols, expected): + assert ax.clean_column_names(cols) == expected + + +def test_click_mapper_maps_pixels_to_steps(): + fig, ax_ = plt.subplots() + ax_.set_xlim(0, 10) + fig.canvas.draw() + mapper = ax.ClickMapper(ax_, step_times=[2.0, 5.0, 8.0]) + for t, expected_step in [(1.0, 0), (3.0, 1), (6.0, 2), (9.0, 3)]: + x_pix, _ = ax_.transData.transform((t, 0)) + assert mapper.to_time(x_pix) == pytest.approx(t) + assert mapper.to_step(x_pix) == expected_step + + +def test_dataclass_defaults(): + assert ax.EpisodeId() == ax.EpisodeId(None, None, None, None) + assert ax.StepId().step is None + info = ax.Info() + assert info.active_tab == "Screenshot" + assert info.exp_result is None + + +# --------------------------------------------------------------------------- Info + + +def _result_df(tmp_path): + return pd.DataFrame( + { + ax.TASK_NAME_KEY: ["t1", "t1", "t2"], + ax.TASK_SEED_KEY: [0, 1, 0], + "agent.agent_name": ["A", "A", "B"], + "cum_reward": [1.0, 0.0, 0.5], + "err_msg": [None, "err", None], + "n_steps": [3, 4, 5], + "exp_dir": [tmp_path / "e0", tmp_path / "e1", tmp_path / "e2"], + } + ) + + +class TestInfo: + def test_get_agent_id(self): + info = ax.Info(agent_id_keys=["agent.agent_name", "x"]) + row = pd.Series({"agent.agent_name": "A", "x": 2, "y": 3}) + assert info.get_agent_id(row) == [("agent.agent_name", "A"), ("x", 2)] + + def test_filter_agent_id_preserves_row_index(self, tmp_path): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.\nagent_name", "A")]) + assert list(info.agent_df.index) == ["t1", "t1"] + assert list(info.agent_df["_row_index"]) == [0, 1] + info.filter_agent_id([("agent.agent_name", "B")]) + assert list(info.agent_df["_row_index"]) == [2] + + def test_update_exp_result(self, tmp_path): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.agent_name", "A")]) + info.step = 5 + info.update_exp_result(ax.EpisodeId("A", "t1", 1, row_index=1)) + assert isinstance(info.exp_result, ExpResult) + assert info.exp_result.exp_dir == tmp_path / "e1" + assert info.step == 0 + + def test_update_exp_result_unknown_row(self, tmp_path): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.agent_name", "A")]) + with pytest.raises(ValueError, match="_row_index: 2"): + info.update_exp_result(ax.EpisodeId("A", "t2", 0, row_index=2)) + assert info.exp_result is None + + def test_update_exp_result_duplicate_rows_warns(self, tmp_path, monkeypatch): + info = ax.Info(result_df=_result_df(tmp_path)) + info.filter_agent_id([("agent.agent_name", "A")]) + info.agent_df["_row_index"] = 0 + warn = MagicMock() + monkeypatch.setattr(ax, "warning", warn) + info.update_exp_result(ax.EpisodeId("A", "t1", 0, row_index=0)) + assert "multiple rows" in warn.call_args[0][0] + assert info.exp_result.exp_dir == tmp_path / "e0" + + @pytest.mark.xfail( + strict=True, + reason="BUG: update_exp_result does not return after the result_df-is-None guard", + ) + def test_update_exp_result_without_results_clears_selection(self): + info = ax.Info() + info.update_exp_result(ax.EpisodeId()) + assert info.exp_result is None + + +# --------------------------------------------------------------------------- markdown + + +class TestDictToMarkdown: + def test_tool_calls_render_empty(self): + assert ax.dict_to_markdown(ToolCalls()) == "" + + def test_non_dict_returns_repr_and_warns(self, monkeypatch): + warn = MagicMock() + monkeypatch.setattr(ax, "warning", warn) + assert ax.dict_to_markdown([1, "a"]) == "[1, 'a']" + warn.assert_called_once() + + def test_empty(self): + assert ax.dict_to_markdown({}) == "No Data" + + def test_nested(self): + d = {"a": 1, "b": {"c": "x"}, "l": [{"k": 2}, "s"], "e": {}} + assert ax.dict_to_markdown(d) == ( + "- **a**: 1\n" + "### b\n- **c**: x\n\n" + "### l\n#### Item 0\n- **k**: 2\n\n- s\n" + "### e\nNo Data\n" + ) + + +class TestDictMsgToMarkdown: + def test_without_role_falls_back(self): + assert ax.dict_msg_to_markdown({"x": 1}) == "- **x**: 1\n" + + def test_content_types(self): + class PydanticLike: + def dict(self): + return {"type": "text", "text": "from dict()"} + + msg = { + "role": "assistant", + "content": [ + {"type": "image", "image": "data:img"}, + {"type": "text", "text": "hello"}, + {"type": "tool_use", "name": "click", "input": {"bid": "a"}, "id": "t1"}, + {"type": "weird", "v": 1}, + PydanticLike(), + ], + } + out = ax.dict_msg_to_markdown(msg) + assert out.startswith("### Assistant\n") + assert "![Image](data:img)" in out + assert "\n```\nhello\n```\n" in out + assert "Tool Call: click `{'bid': 'a'}` (call_id: t1)" in out + assert "{'type': 'weird', 'v': 1}" in out + assert "from dict()" in out + + @pytest.mark.xfail( + strict=True, + reason="BUG: dict messages with plain-string content crash (iterates characters)", + ) + def test_string_content(self): + out = ax.dict_msg_to_markdown({"role": "user", "content": "hello"}) + assert "hello" in out + + +def test_format_tool_call(): + assert ax._format_tool_call("f", '{"a": 1}', "c9") == 'Tool Call: f `{"a": 1}` (call_id: c9)' + + +class TestFormatChatMessage: + def test_langchain_message(self, monkeypatch): + class FakeLCMessage: + def __init__(self, content): + self.content = content + + monkeypatch.setattr(ax, "BaseMessage", FakeLCMessage) + assert ax.format_chat_message(FakeLCMessage("lc content")) == "lc content" + + def test_message_builder(self): + mb = MessageBuilder.assistant().add_text("yo") + assert ax.format_chat_message(mb) == "### Assistant\n\n```\nyo\n```\n" + + def test_message_builder_with_tool_calls(self): + tc = ToolCalls([ToolCall(name="click", arguments={"bid": "a1"}).response_text("ok")]) + md = ax.format_chat_message(MessageBuilder.add_responded_tool_calls(tc)) + assert "**Tool Call 1**: click(bid='a1')" in md + assert "ok" in md + + def test_dict(self): + out = ax.format_chat_message({"role": "user", "content": [{"type": "text", "text": "q"}]}) + assert out == "### User\n\n```\nq\n```\n" + + def test_response_function_tool_call(self): + call = ResponseFunctionToolCall( + name="fill", arguments='{"v": 1}', call_id="c1", type="function_call" + ) + assert ax.format_chat_message(call) == ( + '### Tool Use\n```\nTool Call: fill `{"v": 1}` (call_id: c1)\n```\n' + ) + + def test_other(self): + assert ax.format_chat_message(12) == "12" + assert ax.format_chat_message("No Chat Messages") == "No Chat Messages" + + +def test_page_to_iframe_roundtrip(): + page = "

héllo

" + out = ax._page_to_iframe(page) + b64 = re.search(r'src="data:text/html;base64,([^"]+)"', out).group(1) + assert base64.b64decode(b64).decode("utf-8") == page + assert out.strip().startswith("") + + +def test_code(): + assert ax.code("x") == "```\nx\n```" + + +# --------------------------------------------------------------------------- tabs + + +class TestIfActive: + def test_runs_when_active(self, fresh_info): + fresh_info.active_tab = "Logs" + fn = MagicMock(return_value="ran") + assert ax.if_active("Logs")(fn)(1, k=2) == "ran" + fn.assert_called_once_with(1, k=2) + + @pytest.mark.parametrize("n_out, expected", [(1, GR_UPDATE), (3, (GR_UPDATE,) * 3)]) + def test_skips_when_inactive(self, fresh_info, n_out, expected): + fresh_info.active_tab = "Screenshot" + fn = MagicMock() + assert ax.if_active("Logs", n_out)(fn)() == expected + fn.assert_not_called() + + def test_tab_select_sets_active_tab(self, fresh_info): + ax.tab_select(SimpleNamespace(value="AXTree")) + assert fresh_info.active_tab == "AXTree" + + +class TestStepNavigation: + def test_handle_key_event(self, fresh_info, episode): + ep = ax.EpisodeId("A", "t", 0, 0) + assert ax.handle_key_event("Cmd+Left", ax.StepId(ep, 1)) == ("", ax.StepId(ep, 0)) + assert ax.handle_key_event("Cmd+Left", ax.StepId(ep, 0))[1].step == 0 + assert ax.handle_key_event("Cmd+Right", ax.StepId(ep, 0))[1].step == 1 + # clamps to len(steps) - 2 + assert ax.handle_key_event("Cmd+Right", ax.StepId(ep, 1))[1].step == 1 + assert fresh_info.step == 1 + + def test_handle_key_event_noop(self, fresh_info): + sid = ax.StepId(None, 4) + assert ax.handle_key_event("", sid) == ("", sid) + assert ax.handle_key_event("Shift+Up", sid) == GR_UPDATE + + def test_gallery_step_change(self, fresh_info): + ep = ax.EpisodeId("A") + assert ax.gallery_step_change(SimpleNamespace(index=2), ep) == ax.StepId(ep, 2) + assert fresh_info.step == 2 + + def test_select_step_uses_click_mapper(self, fresh_info): + fresh_info.click_mapper = MagicMock() + fresh_info.click_mapper.to_step.return_value = 3 + out = ax.select_step(ax.EpisodeId("A"), SimpleNamespace(index=(120, 5))) + fresh_info.click_mapper.to_step.assert_called_once_with(120) + assert out.step == 3 and fresh_info.step == 3 + + +# --------------------------------------------------------------------------- screenshots + + +class TestScreenshots: + def test_get_screenshot_raw_and_som(self, fresh_info, episode): + img, action = ax.get_screenshot(fresh_info, step=2) + assert img.getpixel((0, 0)) == (40, 0, 0) and action is None + img, _ = ax.get_screenshot(fresh_info, step=2, som_or_not="SOM Screenshots") + assert img.getpixel((0, 0)) == (0, 40, 0) + + def test_get_screenshot_annotates_with_current_step(self, fresh_info, episode): + img, action = ax.get_screenshot(fresh_info, annotate=True) + assert "" + assert ax.update_pruned_html() == "

" + assert ax.update_axtree() == "```\nAX\n```" + fresh_info.step = 2 + assert ax.update_html() == "No DOM HTML" + assert ax.update_pruned_html() == "No Pruned HTML" + assert ax.update_axtree() == "```\nNo AXTree\n```" + + def test_update_chat_messages_list(self, fresh_info, episode): + out = ax.update_chat_messages() + assert out == "### System\n\n```\nsys\n```\n\n\n### User\n\n```\nuser prompt\n```\n" + + def test_update_chat_messages_default_and_discussion(self, fresh_info, episode): + fresh_info.step = 2 + assert ax.update_chat_messages() == "No Chat Messages" + steps = fresh_info.exp_result.steps_info + steps[2].agent_info["chat_messages"] = Discussion([ax.AgentLabBaseMessage("user", "d")]) + assert ax.update_chat_messages() == "Message 0\n\n```\nd\n```\n\n" + + def test_task_error_and_logs(self, fresh_info, episode, tmp_path): + assert ax.update_task_error() == "```\nTraceback...\n```" + assert ax.update_logs() == "log line" + fresh_info.exp_result = _make_exp_result(tmp_path, [_step(0)], name="bare") + assert ax.update_task_error() == "No Task Error" + assert ax.update_logs() == "No Logs" + + def test_update_stats(self, fresh_info, episode): + df = ax.update_stats() + assert df.to_dict("records") == [ + {"name": "n_tokens", "value": 5}, + {"name": "cost", "value": 0.1}, + ] + fresh_info.step = 9 + assert ax.update_stats() is None + + def test_update_agent_info_md(self, fresh_info, episode): + assert ax.update_agent_info_md() == "# md" + fresh_info.step = 2 + assert "Fill up markdown_page" in ax.update_agent_info_md() + fresh_info.exp_result.steps_info[2].agent_info["markup_page"] = "legacy" + assert ax.update_agent_info_md() == "legacy" + fresh_info.step = 9 + assert ax.update_agent_info_md() is None + + def test_update_agent_info_html(self, fresh_info, episode): + page, s1, s2 = ax.update_agent_info_html() + b64 = re.search(r"base64,([^\"]+)", page).group(1) + assert base64.b64decode(b64).decode() == "hi" + assert s1.getpixel((0, 0)) == (20, 0, 0) and s2.getpixel((0, 0)) == (40, 0, 0) + + fresh_info.exp_result.steps_info[1].agent_info["html_page"] = None + page, _, _ = ax.update_agent_info_html() + assert "Fill up html_page" in page + + fresh_info.step = 9 + assert ax.update_agent_info_html() == (None, None, None) + + @pytest.mark.xfail( + strict=True, + reason="BUG: missing html_page defaults to a list, which _page_to_iframe can't encode", + ) + def test_update_agent_info_html_without_page(self, fresh_info, episode): + fresh_info.step = 2 + page, _, _ = ax.update_agent_info_html() + assert isinstance(page, str) + + def test_update_prompt_tests(self, fresh_info, episode): + prompt = [{"type": "text", "text": "user prompt"}] + assert ax.update_prompt_tests() == (prompt, prompt) + fresh_info.exp_result.steps_info[1].agent_info["chat_messages"] = ["a", "plain"] + assert ax.update_prompt_tests() == ("plain", "plain") + + +class TestSubmitAction: + def _client(self, monkeypatch): + client = MagicMock() + client.chat.completions.create.return_value = SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="LLM says hi"))] + ) + factory = MagicMock(return_value=client) + monkeypatch.setattr(ax, "OpenAI", factory) + return client + + def test_dict_messages(self, fresh_info, episode, monkeypatch): + client = self._client(monkeypatch) + original = fresh_info.exp_result.steps_info[1].agent_info["chat_messages"] + assert ax.submit_action("new prompt") == "LLM says hi" + kwargs = client.chat.completions.create.call_args.kwargs + assert kwargs["model"] == "gpt-4o-mini" + assert kwargs["messages"][1] == {"role": "user", "content": "new prompt"} + assert len(kwargs["messages"]) == 2 + assert original[1]["content"] == [{"type": "text", "text": "user prompt"}] # deep-copied + + def test_langchain_messages_converted(self, fresh_info, episode, monkeypatch): + class FakeBase: + def __init__(self, content): + self.content = content + + class FakeHuman(FakeBase): + pass + + monkeypatch.setattr(ax, "BaseMessage", FakeBase) + monkeypatch.setattr(ax, "HumanMessage", FakeHuman) + fresh_info.exp_result.steps_info[1].agent_info["chat_messages"] = [ + FakeBase("sys"), + FakeHuman("usr"), + ] + client = self._client(monkeypatch) + ax.submit_action("edited") + msgs = client.chat.completions.create.call_args.kwargs["messages"] + assert msgs[0] == {"role": "system", "content": "sys"} + assert msgs[1]["role"] == "user" and msgs[1]["content"] == "edited" + + def test_rejects_non_user_second_message(self, fresh_info, episode, monkeypatch): + self._client(monkeypatch) + fresh_info.exp_result.steps_info[1].agent_info["chat_messages"] = [ + {"role": "system", "content": "a"}, + {"role": "assistant", "content": "b"}, + ] + with pytest.raises(AssertionError, match="Second message should be user"): + ax.submit_action("x") + + def test_rejects_unknown_message_types(self, fresh_info, episode, monkeypatch): + self._client(monkeypatch) + fresh_info.exp_result.steps_info[1].agent_info["chat_messages"] = ["a", "b"] + with pytest.raises(ValueError, match="list of BaseMessage or dict"): + ax.submit_action("x") + + +# --------------------------------------------------------------------------- step info panels + + +class TestStepInfoPanels: + def test_update_step_info(self, fresh_info, episode): + ep_info, action_info, state_error = ax.update_step_info() + assert "### task.a (seed: 3)" in ep_info + assert "### Step 1 / 2 (Reward: 1.0)" in ep_info + assert "Do it" in ep_info + assert "Terminated: None, Truncated: None" in ep_info + assert "study/exp" in ep_info + assert action_info.startswith("**Action:**") + assert " & go", + "chat_messages": [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": [{"type": "text", "text": XSS}]}, + ], + }, + ), + (_obs(2), "noop()", {}), + ] + return _make_exp_result(tmp_path, steps) + + +class TestExpResultToHtml: + def test_full_episode_embedded(self, full_exp): + html = e2h.exp_result_to_html(full_exp) + + assert '

miniwob.click-test
' in html + assert '
Seed: 7
' in html + assert "Click the button <script>alert('x')</script>" in html + assert XSS not in html + + # reset step with no action skipped, others rendered + assert "Step 0" not in html + assert "Step 1" in html + assert "Step 2" in html + assert "click('a1')" in html + assert "I should <click> & go" in html + + # screenshots embedded for step 1 and 2 (main + som each) + assert html.count('src="data:image/png;base64,') == 4 + assert "file://" not in html + assert "Screenshot_som[1]" in html + + assert "AXTree" in html + assert "[a1] button 'OK' <b>" in html + assert "HTML" in html + assert "<div id='x'>hi</div>" in html + assert "Prompt" in html + assert '
SYSTEM
' in html + assert '
USER
' in html + + # step 2 has no think/axtree/prompt -> sections only appear once + assert html.count("AXTree") == 1 + assert html.count("Think:") == 1 + assert html.count("Prompt") == 1 + + def test_embedded_image_roundtrips(self, full_exp): + html = e2h.exp_result_to_html(full_exp) + b64 = ( + re.search(r'alt="Screenshot 1"', html) + and re.findall(r'src="data:image/png;base64,([^"]+)" alt="Screenshot 1"', html)[0] + ) + img = Image.open(BytesIO(base64.b64decode(b64))) + assert img.size == (8, 6) + assert img.convert("RGB").getpixel((0, 0)) == (10, 0, 0) + + def test_paths_instead_of_embedding(self, full_exp): + html = e2h.exp_result_to_html(full_exp, embed_images=False) + assert "data:image/png" not in html + main = full_exp.exp_dir.resolve() / "screenshot_step_1.png" + som = full_exp.exp_dir.resolve() / "screenshot_som_step_1.png" + assert f'src="file://{main}" alt="Screenshot 1"' in html + assert f'src="file://{som}" alt="SOM Screenshot 1"' in html + + @pytest.mark.parametrize( + "kwargs, n_open_nested, step_open", + [ + ({}, 0, True), + ({"steps_open": False}, 0, False), + ({"som_open": True}, 2, True), + ({"axtree_open": True, "html_open": True, "prompt_open": True}, 3, True), + ( + {"som_open": True, "axtree_open": True, "html_open": True, "prompt_open": True}, + 5, + True, + ), + ], + ) + def test_open_flags(self, full_exp, kwargs, n_open_nested, step_open): + html = e2h.exp_result_to_html(full_exp, **kwargs) + n_open = len(re.findall(r'
', html)) + assert n_open == n_open_nested + assert html.count("
") == (2 if step_open else 0) + assert html.count("
") == (0 if step_open else 2) + + def test_missing_screenshots(self, tmp_path): + steps = [ + (_obs(0, goal="g", with_screens=False), None, {}), + (_obs(1, with_screens=False), "noop()", {}), + ] + exp = _make_exp_result(tmp_path, steps, som=False) + for embed in (True, False): + html = e2h.exp_result_to_html(exp, embed_images=embed) + assert "

Screenshot not available

" in html + assert "Screenshot_som" not in html + + def test_first_step_with_action_is_rendered(self, tmp_path): + exp = _make_exp_result(tmp_path, [(_obs(0, goal="g"), "noop()", {})]) + html = e2h.exp_result_to_html(exp) + assert "Step 0" in html + assert "noop()" in html + + def test_no_steps(self): + exp = MagicMock(spec=ExpResult) + exp.exp_args.env_args = SimpleNamespace(task_name="t", task_seed=1) + exp.steps_info = [] + html = e2h.exp_result_to_html(exp) + assert "goal-section" not in html.split("")[1] + assert "") + + def test_goal_lookup_failure_is_tolerated(self): + exp = MagicMock(spec=ExpResult) + exp.exp_args.env_args = SimpleNamespace(task_name="t", task_seed=1) + exp.steps_info = [StepInfo(step=0, obs=None, action=None)] + html = e2h.exp_result_to_html(exp) + assert '
' not in html + + def test_missing_goal_key_uses_default(self): + exp = MagicMock(spec=ExpResult) + exp.exp_args.env_args = SimpleNamespace(task_name="t", task_seed=1) + exp.steps_info = [StepInfo(step=0, obs={}, action=None)] + html = e2h.exp_result_to_html(exp) + assert "
No goal specified
" in html + + @pytest.mark.xfail(strict=True, reason="BUG: task_name is interpolated into HTML unescaped") + def test_task_name_is_escaped(self): + exp = MagicMock(spec=ExpResult) + exp.exp_args.env_args = SimpleNamespace(task_name=XSS, task_seed=1) + exp.steps_info = [] + assert XSS not in e2h.exp_result_to_html(exp) + + +class TestScreenshotHelpers: + def test_screenshot_index_error(self): + exp = MagicMock() + exp.get_screenshot.side_effect = IndexError + exp.get_screenshot_path.side_effect = IndexError + assert e2h._get_screenshot_html(exp, 3, True) == "

Screenshot not available

" + assert e2h._get_screenshot_html(exp, 3, False) == "

Screenshot not available

" + assert e2h._get_som_screenshot_html(exp, 3, True, True) == "" + assert e2h._get_som_screenshot_html(exp, 3, True, False) == "" + + def test_som_none_image_returns_empty(self): + exp = MagicMock() + exp.get_screenshot.return_value = None + assert e2h._get_som_screenshot_html(exp, 0, False, True) == "" + + def test_som_nonexistent_path_returns_empty(self, tmp_path): + exp = MagicMock() + exp.get_screenshot_path.return_value = tmp_path / "nope.png" + assert e2h._get_som_screenshot_html(exp, 0, False, False) == "" + exp.get_screenshot_path.return_value = None + assert e2h._get_som_screenshot_html(exp, 0, False, False) == "" + + def test_som_embedded_open(self): + exp = MagicMock() + exp.get_screenshot.return_value = Image.new("RGB", (2, 2)) + html = e2h._get_som_screenshot_html(exp, 4, True, True) + exp.get_screenshot.assert_called_once_with(4, som=True) + assert '
' in html + assert "Screenshot_som[4]" in html + assert 'alt="SOM Screenshot 4"' in html + + def test_image_to_html(self): + assert e2h._image_to_html(None, "Pic") == "

Pic not available

" + img = Image.new("RGB", (3, 2), (1, 2, 3)) + out = e2h._image_to_html(img, "Pic") + m = re.fullmatch( + r'Pic', out + ) + assert m + decoded = Image.open(BytesIO(base64.b64decode(m.group(1)))) + assert decoded.size == (3, 2) and decoded.getpixel((0, 0)) == (1, 2, 3) + + def test_path_to_html(self, tmp_path, monkeypatch): + assert e2h._path_to_html(None, "P") == "

P not available

" + assert e2h._path_to_html(tmp_path / "missing.png", "P") == "

P not available

" + f = tmp_path / "a.png" + f.write_bytes(b"x") + monkeypatch.chdir(tmp_path) + out = e2h._path_to_html(Path("a.png"), "P") + assert out == f'P' + + +class TestFormatGoal: + def test_none(self): + assert e2h._format_goal(None) == "
No goal specified
" + + def test_string_escaped(self): + assert e2h._format_goal("a < b") == "
a < b
" + + def test_multipart_keeps_only_text(self): + goal = [ + {"type": "text", "text": "line1"}, + {"type": "image_url", "image_url": {"url": "data:..."}}, + {"type": "text", "text": "line2"}, + ] + assert e2h._format_goal(goal) == "
line1\nline2
" + + +class TestFormatChatMessages: + def test_empty(self): + assert e2h._format_chat_messages_like_xray([]) == ( + "
No chat messages
" + ) + assert "No chat messages" in e2h._format_chat_messages_like_xray(None) + + def test_object_with_role_and_content(self): + msg = SimpleNamespace(role="assistant", content="") + out = e2h._format_chat_messages_like_xray([msg]) + assert out == ( + '
ASSISTANT
' + '
<hi>
' + ) + + def test_message_builder(self): + mb = MessageBuilder.user().add_text("hello") + out = e2h._format_chat_messages_like_xray([mb]) + assert '
USER
' in out + assert "hello" in out + + def test_agentlab_base_message_is_dict(self): + out = e2h._format_chat_messages_like_xray([BaseMessage("user", "plain")]) + assert out == ( + '
USER
' + '
plain
' + ) + + def test_dict_multipart(self): + msg = { + "role": "assistant", + "content": [ + {"type": "text", "text": "a&b"}, + {"type": "image", "image": "data:png"}, + {"type": "tool_use", "name": "click", "input": {"bid": "a1"}, "call_id": "c1"}, + {"type": "other", "x": 1}, + "raw ", + ], + } + out = e2h._format_chat_messages_like_xray([msg]) + blocks = re.findall(r'
(.*?)
', out) + assert blocks == [ + "a&b", + "[IMAGE]", + "Tool Call: click `{'bid': 'a1'}` (call_id: c1)", + "{'type': 'other', 'x': 1}", + "raw <part>", + ] + + def test_dict_defaults(self): + out = e2h._format_chat_messages_like_xray([{"foo": 1}]) + assert '
UNKNOWN
' in out + assert "{'foo': 1}" in out + + def test_other_types(self): + out = e2h._format_chat_messages_like_xray(["", 42]) + assert out.count('
') == 2 + assert "<str msg>" in out and ">42<" in out + + @pytest.mark.xfail(strict=True, reason="BUG: chat message role is not HTML-escaped") + def test_role_is_escaped(self): + out = e2h._format_chat_messages_like_xray([{"role": XSS, "content": "x"}]) + assert "")' + a1 = _arg("bid", "a1", 5, 9) + html = overlay_utils.create_colored_html(s, [a1], [(1.0, 0, 0)]) + assert "