diff --git a/src/agentlab/analyze/inspect_results.py b/src/agentlab/analyze/inspect_results.py index a66848355..eb9703c2d 100644 --- a/src/agentlab/analyze/inspect_results.py +++ b/src/agentlab/analyze/inspect_results.py @@ -560,7 +560,9 @@ def map_err_key(err_msg: str): return err_msg # remove logs from the message if any - err_msg = err_msg[: err_msg.find("=== logs ===")].rstrip() + logs_idx = err_msg.find("=== logs ===") + if logs_idx >= 0: + err_msg = err_msg[:logs_idx].rstrip() regex_replacements = [ ( r"your messages resulted in \d+ tokens", diff --git a/tests/analyze/test_inspect_results_stats.py b/tests/analyze/test_inspect_results_stats.py new file mode 100644 index 000000000..2d5c8c872 --- /dev/null +++ b/tests/analyze/test_inspect_results_stats.py @@ -0,0 +1,249 @@ +"""Unit tests for the statistics and error categorization helpers of inspect_results.""" + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from agentlab.analyze import inspect_results +from agentlab.analyze.inspect_results import ( + error_report, + get_sample_std_err, + get_std_err, + map_err_key, + summarize, +) + + +def make_result_df(cum_reward, err_msg=None, n_steps=None, **extra): + n = len(cum_reward) + data = { + "cum_reward": cum_reward, + "err_msg": err_msg if err_msg is not None else [None] * n, + "n_steps": n_steps if n_steps is not None else [1] * n, + } + data.update(extra) + return pd.DataFrame(data) + + +def test_get_std_err_binary_uses_bernoulli_formula(): + df = make_result_df([1.0, 1.0, 0.0, 0.0, 1.0]) + mean, std_err = get_std_err(df, "cum_reward") + + expected_mean = 3 / 5 + assert mean == pytest.approx(expected_mean) + assert std_err == pytest.approx(np.sqrt(expected_mean * (1 - expected_mean) / 5)) + + +def test_get_std_err_binary_ignores_missing_values(): + df = make_result_df([1.0, 0.0, np.nan, 1.0]) + mean, std_err = get_std_err(df, "cum_reward") + + assert mean == pytest.approx(2 / 3) + assert std_err == pytest.approx(np.sqrt((2 / 3) * (1 / 3) / 3)) + + +def test_get_std_err_all_success_has_zero_std_err(): + df = make_result_df([1.0, 1.0, 1.0]) + mean, std_err = get_std_err(df, "cum_reward") + + assert mean == pytest.approx(1.0) + assert std_err == pytest.approx(0.0) + + +def test_get_std_err_non_binary_falls_back_to_sample_std_err(): + df = make_result_df([0.2, 0.5, 1.0]) + + assert get_std_err(df, "cum_reward") == get_sample_std_err(df, "cum_reward") + + +def test_get_sample_std_err_matches_sample_formula(): + values = [0.2, 0.5, 1.0, 0.7] + df = make_result_df(values) + mean, std_err = get_sample_std_err(df, "cum_reward") + + assert mean == pytest.approx(np.mean(values)) + assert std_err == pytest.approx(np.std(values, ddof=1) / np.sqrt(len(values))) + + +def test_get_sample_std_err_singleton_normalizes_nan_to_zero(): + df = make_result_df([0.42]) + mean, std_err = get_sample_std_err(df, "cum_reward") + + assert mean == pytest.approx(0.42) + assert std_err == 0 + assert not np.isnan(std_err) + + +def test_get_std_err_singleton_non_binary_normalizes_nan_to_zero(): + df = make_result_df([0.42]) + _, std_err = get_std_err(df, "cum_reward") + + assert std_err == 0 + + +def test_summarize_mixed_success_and_error(): + df = make_result_df( + cum_reward=[1.0, 0.0, 0.0, 1.0], + err_msg=[None, None, "boom", None], + n_steps=[3, 5, 1, 7], + terminated=[True, True, False, True], + truncated=[False, False, False, False], + ) + record = summarize(df) + + assert record["avg_reward"] == pytest.approx(0.5) + assert record["std_err"] == pytest.approx(round(np.sqrt(0.5 * 0.5 / 4), 3)) + assert record["avg_steps"] == pytest.approx(4.0) + assert record["n_completed"] == "4/4" + assert record["n_err"] == 1 + + +def test_summarize_counts_incomplete_episodes(): + df = make_result_df( + cum_reward=[1.0, 0.0, 0.0], + err_msg=[None, "boom", None], + terminated=[True, False, False], + truncated=[False, False, False], + ) + record = summarize(df) + + # the errored episode counts as completed, the unfinished one does not + assert record["n_completed"] == "2/3" + assert record["n_err"] == 1 + + +def test_summarize_returns_none_when_nothing_completed(): + df = make_result_df( + cum_reward=[0.0, 0.0], + terminated=[False, False], + truncated=[False, False], + ) + assert summarize(df) is None + + +def test_summarize_without_cum_reward_column(): + df = pd.DataFrame({"err_msg": [None, None], "n_steps": [1, 2]}) + record = summarize(df) + + assert np.isnan(record["avg_reward"]) + assert np.isnan(record["std_err"]) + assert record["n_completed"] == "0/2" + assert record["n_err"] == 0 + + +def test_summarize_rejects_non_zero_reward_on_error(): + df = make_result_df( + cum_reward=[1.0, 1.0], + err_msg=[None, "boom"], + terminated=[True, True], + ) + with pytest.raises(AssertionError): + summarize(df) + + +def test_summarize_reports_cum_cost(): + df = make_result_df( + cum_reward=[1.0, 0.0], + terminated=[True, True], + **{"stats.cum_cost": [0.125, 0.25]}, + ) + record = summarize(df) + + assert record["cum_cost"] == pytest.approx(0.375) + + +def test_summarize_effective_cost_replaces_cum_cost(): + df = make_result_df( + cum_reward=[1.0, 0.0], + terminated=[True, True], + **{"stats.cum_cost": [0.125, 0.25], "stats.cum_effective_cost": [0.1, 0.2]}, + ) + record = summarize(df) + + assert record["cum_effective_cost"] == pytest.approx(0.3) + assert "cum_cost" not in record + + +def test_map_err_key_passes_through_none(): + assert map_err_key(None) is None + + +def test_map_err_key_keeps_message_without_logs(): + assert map_err_key("ValueError: bad thing") == "ValueError: bad thing" + + +def test_map_err_key_strips_logs(): + err_msg = "ValueError: bad thing\n=== logs ===\nstep 1\nstep 2" + assert map_err_key(err_msg) == "ValueError: bad thing" + + +def test_map_err_key_normalizes_token_counts(): + key_a = map_err_key("Error: your messages resulted in 12345 tokens") + key_b = map_err_key("Error: your messages resulted in 999 tokens") + + assert key_a == key_b == "Error: your messages resulted in x tokens" + + +def test_map_err_key_normalizes_task_names(): + key_a = map_err_key("Exception uncaught by agent or environment in task webarena.42.") + key_b = map_err_key("Exception uncaught by agent or environment in task webarena.7.") + + assert key_a == key_b == "Exception uncaught by agent or environment in task ." + + +def _fake_exp_result(task_name, task_seed, stack_trace): + return SimpleNamespace( + exp_dir=f"/tmp/{task_name}_{task_seed}", + exp_args=SimpleNamespace( + env_args=SimpleNamespace(task_name=task_name, task_seed=task_seed) + ), + summary_info={"stack_trace": stack_trace}, + ) + + +def test_error_report_buckets_equivalent_errors(monkeypatch): + exp_results = { + "/exp/0": _fake_exp_result("task_b", 0, "trace 0"), + "/exp/1": _fake_exp_result("task_a", 1, "trace 1"), + "/exp/2": _fake_exp_result("task_c", 2, "trace 2"), + } + monkeypatch.setattr(inspect_results, "get_exp_result", lambda exp_dir: exp_results[exp_dir]) + + df = pd.DataFrame( + { + "exp_dir": ["/exp/0", "/exp/1", "/exp/2"], + "err_msg": [ + "Error: your messages resulted in 10 tokens\n=== logs ===\nnoise", + "Error: your messages resulted in 20 tokens", + "ValueError: something else", + ], + } + ) + report = error_report(df) + + # the two token-count errors collapse into a single bucket + assert "## 2x : Error: your messages resulted in x tokens" in report + assert "## 1x : ValueError: something else" in report + # tasks of a bucket are listed sorted by task name + assert report.index("* task_a seed: 1") < report.index("* task_b seed: 0") + assert "Stack Trace: \n trace 2" in report + + +def test_error_report_limits_stack_traces(monkeypatch): + exp_results = {f"/exp/{i}": _fake_exp_result(f"task_{i}", i, f"trace {i}") for i in range(3)} + monkeypatch.setattr(inspect_results, "get_exp_result", lambda exp_dir: exp_results[exp_dir]) + + df = pd.DataFrame( + { + "exp_dir": list(exp_results.keys()), + "err_msg": ["boom"] * 3, + } + ) + report = error_report(df, max_stack_trace=2) + + assert report.count("Stack Trace:") == 2 + # all tasks of the bucket are still listed + for i in range(3): + assert f"* task_{i} seed: {i}" in report