From 6eda48b8e766692bfbab1cbbe1973ff230afaf10 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Wed, 4 Nov 2026 11:33:33 +0000 Subject: [PATCH 1/3] fix(agent): mark agent infrastructure and simulated-user errors as unscored Agent CLI quota and server errors, and simulated-user errors, now set generated_error instead of being scored as failures. A simulated-user error no longer skips the remaining scenarios of an evalset. The analyzer leaves rows with generated_error out of non-binary averages, because scorers return 0 for them. --- evalbench/evaluator/agentevaluator.py | 55 ++++++++-- evalbench/reporting/analyzer.py | 7 ++ evalbench/test/agent_infra_errors_test.py | 119 ++++++++++++++++++++++ evalbench/work/agentscorework.py | 2 +- 4 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 evalbench/test/agent_infra_errors_test.py diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index e81fd5562..bee5b807c 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -16,10 +16,32 @@ from evaluator.simulateduser import SimulatedUser from work.agentscorework import AgentScoreWork import json +import re import subprocess from typing import Dict +# Quota and server errors in agent CLI stderr. The tool under test cannot +# cause these, so the trial is unscored instead of failed. Status codes must +# follow a word such as "HTTP" or "error", so a traceback line number such as +# "line 503" does not match. +_INFRA_ERROR_RE = re.compile( + r"\b(?:HTTP|[Ss]tatus|[Cc]ode|[Ee]rror)\b\D{0,10}\b(?:429|50[0234])\b" + r"|Too Many Requests|RESOURCE_EXHAUSTED|UNAVAILABLE" + r"|Internal Server Error|Service Unavailable" + r"|[Qq]uota[^\n]{0,40}exceeded|exceeded[^\n]{0,40}quota") + + +def _infra_error(result: Any) -> str | None: + """Returns an error message if the agent CLI failed on quota or a server error.""" + returncode = getattr(result, "returncode", 0) + stderr = getattr(result, "stderr", "") or "" + # 124 is a timeout, which counts as a failure. + if returncode in (0, 124) or not _INFRA_ERROR_RE.search(stderr): + return None + return f"Agent infrastructure error (exit {returncode}): {stderr.strip()[-500:]}" + + class AgentEvaluator: def __init__( self, @@ -162,6 +184,7 @@ def process_scenario( else self.eval_case_timeout_seconds ) last_result = None + generated_error = None for turn in range(max_turns): remaining_timeout_seconds = None @@ -241,13 +264,27 @@ def process_scenario( "agent": result.stdout }) + generated_error = _infra_error(result) + if generated_error: + logging.error(generated_error) + break + if turn < max_turns - 1: if simulated_user: - next_response = simulated_user.get_next_response( - conversation_plan, - conversation_history, - result.stdout - ) + try: + next_response = simulated_user.get_next_response( + conversation_plan, + conversation_history, + result.stdout + ) + except Exception as e: + generated_error = f"Simulated user error: {e}" + logging.error(generated_error) + break + if not str(next_response or "").strip(): + generated_error = "Simulated user returned an empty reply." + logging.error(generated_error) + break if "TERMINATE" in next_response: logging.info("Simulated user terminated conversation.") break @@ -268,7 +305,8 @@ def process_scenario( accumulated_skills, eval_result, job_id, - metadata + metadata, + generated_error, ) def _log_cli_result(self, turn: int, max_turns: int, result: subprocess.CompletedProcess): @@ -289,7 +327,8 @@ def _finalize_scenario( accumulated_skills: List[str], eval_result: Any, job_id: str, - metadata: Dict[str, Any] + metadata: Dict[str, Any], + generated_error: str | None = None, ): """Finalizes the scenario by scoring and appending results.""" timed_out = ( @@ -305,7 +344,7 @@ def _finalize_scenario( "returncode": getattr(last_result, "returncode", 0), "timed_out": timed_out, "prompt_generator_error": None, - "generated_error": None, + "generated_error": generated_error, "sql_generator_error": None, "golden_error": None, "generated_sql": "skipped", diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index 6068355ad..b047b248c 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -220,6 +220,13 @@ def analyze_one_metric( "skills_best_practices", ] if metric_name in non_binary_metrics: + # Scorers return 0 for a trial with a generated_error, such as + # an agent quota error. Leave those rows out of the average. + if "generated_error" in df_metric.columns: + df_metric = df_metric[ + df_metric["generated_error"].isna() + | (df_metric["generated_error"] == "") + ] avg_val = ( df_metric["score"].mean() if (not df_metric.empty and "score" in df_metric.columns) diff --git a/evalbench/test/agent_infra_errors_test.py b/evalbench/test/agent_infra_errors_test.py new file mode 100644 index 000000000..71748a1eb --- /dev/null +++ b/evalbench/test/agent_infra_errors_test.py @@ -0,0 +1,119 @@ +import subprocess +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from evaluator.agentevaluator import AgentEvaluator, _infra_error +from generators.models.agent_cli import AgentCliGenerator +from reporting.analyzer import analyze_one_metric +from work.agentscorework import AgentScoreWork + + +def _result(returncode, stderr=""): + return subprocess.CompletedProcess([], returncode, "agent output", stderr) + + +@pytest.mark.parametrize("returncode, stderr, is_infra", [ + (1, "Error: 429 Too Many Requests", True), + (1, "status: RESOURCE_EXHAUSTED", True), + (1, "HTTP 503 Service Unavailable", True), + (1, "API status code 502", True), + (1, "Quota exceeded for aiplatform requests", True), + (124, "TimeoutError after 503 seconds", False), + (0, "Error 429", False), + (1, "Tool failed: table not found", False), + (1, 'File "tool.py", line 503, in run', False), + (1, "WARNING: credentials without a quota project", False), +], ids=["429", "resource_exhausted", "503", "status_code_502", + "quota_exceeded", "timeout", "success", "agent_failure", + "traceback_line_number", "quota_project_warning"]) +def test_infra_error_detection(returncode, stderr, is_infra): + assert bool(_infra_error(_result(returncode, stderr))) == is_infra + + +@pytest.fixture +def run_scenario(): + """Runs one 3-turn scenario and returns (generated_error, simulated_user).""" + patcher = patch("evaluator.agentevaluator.get_generator") + generator = MagicMock(spec=AgentCliGenerator) + generator.version = "1.0" + generator.name = "mock_cli" + generator.fake_home = None + generator.parse_response.return_value = {} + generator.extract_tools.return_value = [] + generator.extract_skills.return_value = [] + patcher.start().return_value = generator + + def _run(cli_result, simulated_user_reply): + generator.safe_generate.return_value = cli_result + simulated_user = MagicMock() + if isinstance(simulated_user_reply, Exception): + simulated_user.get_next_response.side_effect = simulated_user_reply + else: + simulated_user.get_next_response.return_value = simulated_user_reply + evaluator = AgentEvaluator({"model_config": "dummy.yaml"}) + evaluator._finalize_scenario = MagicMock() + evaluator.process_scenario( + {"id": "s1", "starting_prompt": "hi", "max_turns": 3}, + eval_result=MagicMock(), job_id="job1", metadata={}, + simulated_user=simulated_user) + return evaluator._finalize_scenario.call_args.args[-1], simulated_user + + yield _run + patcher.stop() + + +def test_infra_error_stops_the_conversation(run_scenario): + error, simulated_user = run_scenario(_result(1, "429 Too Many Requests"), + "next") + assert error.startswith("Agent infrastructure error (exit 1)") + simulated_user.get_next_response.assert_not_called() + + +@pytest.mark.parametrize("reply, expected", [ + (RuntimeError("model down"), "Simulated user error: model down"), + (" ", "Simulated user returned an empty reply."), +], ids=["raises", "empty_reply"]) +def test_simulated_user_problem_is_recorded(run_scenario, reply, expected): + error, _ = run_scenario(_result(0), reply) + assert error == expected + + +def test_normal_run_has_no_error(run_scenario): + error, _ = run_scenario(_result(0), "TERMINATE") + assert error is None + + +def test_score_work_passes_generated_error(): + eval_output = {"eval_id": "s1", "generated_error": "Agent infrastructure error", + "scenario": {}, "metadata": {}} + with patch("scorers.score.compare") as mock_compare: + AgentScoreWork(config={}, eval_output=eval_output, + scoring_results=[]).run() + item = mock_compare.call_args.kwargs["eval_output_item"] + assert item["generated_error"] == "Agent infrastructure error" + + +@pytest.mark.parametrize("generated_error", ["Agent infrastructure error", None]) +def test_finalize_scenario_writes_generated_error(generated_error): + with patch("evaluator.agentevaluator.get_generator") as mock_get_generator: + mock_get_generator.return_value = MagicMock(spec=AgentCliGenerator) + evaluator = AgentEvaluator({"model_config": "dummy.yaml"}) + eval_result = MagicMock(agent_results=[], scoring_results=[]) + with patch("evaluator.agentevaluator.AgentScoreWork"): + evaluator._finalize_scenario( + {"id": "s1", "starting_prompt": "hi"}, _result(1), [], [], [], + eval_result, "job1", {}, generated_error) + assert eval_result.agent_results[0]["generated_error"] == generated_error + + +def test_average_skips_rows_with_generated_error(): + df = pd.DataFrame([ + {"comparator": "end_to_end_latency", "score": 1000.0, + "generated_error": None}, + {"comparator": "end_to_end_latency", "score": 0.0, + "generated_error": "Agent infrastructure error"}, + ]) + summary = analyze_one_metric(df, "end_to_end_latency", 100) + assert summary["metric_score"] == 1000.0 diff --git a/evalbench/work/agentscorework.py b/evalbench/work/agentscorework.py index d3ec0d44a..6ffb69477 100644 --- a/evalbench/work/agentscorework.py +++ b/evalbench/work/agentscorework.py @@ -51,7 +51,7 @@ def run(self, work_config: Any = None) -> Any: "generated_sql": "skipped", "generated_result": self.eval_output.get("accumulated_tools", []), "eval_results": self.eval_output, - "generated_error": None, + "generated_error": self.eval_output.get("generated_error"), "dialects": metadata.get("dialects", []), "database": metadata.get("database", "unknown"), "job_id": self.eval_output.get("job_id"), From 3a6992ed9b94f02c9ec5e8c31390150fd22f8d41 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Wed, 4 Nov 2026 13:38:12 +0000 Subject: [PATCH 2/3] fix(agent): let an empty simulated-user reply continue the conversation The noop simulated user of the agent gRPC proxy flow returns an empty reply by design, so treating it as an error stopped multi-turn proxy scenarios after turn 1. Only a simulated-user exception now sets generated_error. --- evalbench/evaluator/agentevaluator.py | 4 ---- evalbench/test/agent_infra_errors_test.py | 17 ++++++++++------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index bee5b807c..f81f50ad3 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -281,10 +281,6 @@ def process_scenario( generated_error = f"Simulated user error: {e}" logging.error(generated_error) break - if not str(next_response or "").strip(): - generated_error = "Simulated user returned an empty reply." - logging.error(generated_error) - break if "TERMINATE" in next_response: logging.info("Simulated user terminated conversation.") break diff --git a/evalbench/test/agent_infra_errors_test.py b/evalbench/test/agent_infra_errors_test.py index 71748a1eb..dfa04ab50 100644 --- a/evalbench/test/agent_infra_errors_test.py +++ b/evalbench/test/agent_infra_errors_test.py @@ -71,13 +71,16 @@ def test_infra_error_stops_the_conversation(run_scenario): simulated_user.get_next_response.assert_not_called() -@pytest.mark.parametrize("reply, expected", [ - (RuntimeError("model down"), "Simulated user error: model down"), - (" ", "Simulated user returned an empty reply."), -], ids=["raises", "empty_reply"]) -def test_simulated_user_problem_is_recorded(run_scenario, reply, expected): - error, _ = run_scenario(_result(0), reply) - assert error == expected +def test_simulated_user_error_is_recorded(run_scenario): + error, _ = run_scenario(_result(0), RuntimeError("model down")) + assert error == "Simulated user error: model down" + + +def test_empty_simulated_user_reply_continues(run_scenario): + """The noop simulated user of proxy flows returns "" by design.""" + error, simulated_user = run_scenario(_result(0), "") + assert error is None + assert simulated_user.get_next_response.call_count == 2 def test_normal_run_has_no_error(run_scenario): From 7aa1a72036345984b6ef2d0a55579221c447b633 Mon Sep 17 00:00:00 2001 From: Omkar Gaikwad Date: Fri, 6 Nov 2026 16:04:50 +0000 Subject: [PATCH 3/3] fix(agent): detect model API errors per agent CLI, not by scanning stderr The stderr regex marked real failures as infrastructure errors: Gemini CLI retry logs, errors from the tool under test, and gRPC proxy error messages. It also missed errors that Claude Code and Codex CLI write to stdout. AgentCliGenerator.model_api_error() now returns the final model API error if it is a quota or server error. Gemini CLI reads the "[API Error: ...]" result event, Claude Code reads the rate_limit and server_error categories of top-level assistant messages, and Codex CLI reads the turn.failed message. The default returns None, so agy_cli and agent_grpc_proxy errors still count as failures. An empty or None simulated-user reply is now an error, because QueryGenerator.generate returns "" after a quota error and ClaudeGenerator returns None. The noop simulated users still continue the conversation. --- evalbench/evaluator/agentevaluator.py | 40 +++-- evalbench/generators/models/agent_cli.py | 10 ++ evalbench/generators/models/claude_code.py | 20 +++ evalbench/generators/models/codex_cli.py | 25 +++ evalbench/generators/models/gemini_cli.py | 30 ++++ evalbench/reporting/analyzer.py | 3 +- evalbench/test/agent_infra_errors_test.py | 188 ++++++++++++++++++--- evalbench/test/agentevaluator_test.py | 1 + evalbench/test/test_eval_case_timeout.py | 6 + 9 files changed, 278 insertions(+), 45 deletions(-) diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index f81f50ad3..5dd6a3320 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -10,36 +10,34 @@ from dataset.evalgeminicliinput import EvalGeminiCliRequest from generators.models import get_generator from generators.models.agent_cli import AgentCliGenerator +from generators.models.noop_agent import NoopAgentGenerator +from generators.models.passthrough import NOOPGenerator from mp import mprunner from util.config import get_eval_case_timeout from work.agentgenwork import AgentGenWork from evaluator.simulateduser import SimulatedUser from work.agentscorework import AgentScoreWork import json -import re import subprocess from typing import Dict -# Quota and server errors in agent CLI stderr. The tool under test cannot -# cause these, so the trial is unscored instead of failed. Status codes must -# follow a word such as "HTTP" or "error", so a traceback line number such as -# "line 503" does not match. -_INFRA_ERROR_RE = re.compile( - r"\b(?:HTTP|[Ss]tatus|[Cc]ode|[Ee]rror)\b\D{0,10}\b(?:429|50[0234])\b" - r"|Too Many Requests|RESOURCE_EXHAUSTED|UNAVAILABLE" - r"|Internal Server Error|Service Unavailable" - r"|[Qq]uota[^\n]{0,40}exceeded|exceeded[^\n]{0,40}quota") - - -def _infra_error(result: Any) -> str | None: - """Returns an error message if the agent CLI failed on quota or a server error.""" +def _infra_error(generator: AgentCliGenerator, result: Any) -> str | None: + """Returns an error message if the model API hit a quota or server error.""" returncode = getattr(result, "returncode", 0) - stderr = getattr(result, "stderr", "") or "" # 124 is a timeout, which counts as a failure. - if returncode in (0, 124) or not _INFRA_ERROR_RE.search(stderr): + if returncode == 124: + return None + api_error = generator.model_api_error(result) + if not api_error: return None - return f"Agent infrastructure error (exit {returncode}): {stderr.strip()[-500:]}" + return f"Agent infrastructure error (exit {returncode}): {api_error.strip()[-500:]}" + + +def _is_noop(simulated_user: SimulatedUser) -> bool: + """Returns True if the simulated user returns an empty reply by design.""" + return isinstance(simulated_user.model_generator, + (NOOPGenerator, NoopAgentGenerator)) class AgentEvaluator: @@ -264,7 +262,7 @@ def process_scenario( "agent": result.stdout }) - generated_error = _infra_error(result) + generated_error = _infra_error(self.generator, result) if generated_error: logging.error(generated_error) break @@ -277,6 +275,12 @@ def process_scenario( conversation_history, result.stdout ) + # QueryGenerator.generate returns "" when out of quota. + # ClaudeGenerator returns None on any error. + if (not str(next_response or "").strip() + and not _is_noop(simulated_user)): + raise RuntimeError( + "empty reply from the simulated user model") except Exception as e: generated_error = f"Simulated user error: {e}" logging.error(generated_error) diff --git a/evalbench/generators/models/agent_cli.py b/evalbench/generators/models/agent_cli.py index e8d9e2e94..3f167087b 100644 --- a/evalbench/generators/models/agent_cli.py +++ b/evalbench/generators/models/agent_cli.py @@ -114,3 +114,13 @@ def extract_tools(self, stdout: str) -> list: @abstractmethod def extract_skills(self, stdout: str) -> list: raise NotImplementedError("Subclasses must implement this method") + + def model_api_error( + self, result: subprocess.CompletedProcess + ) -> Optional[str]: + """Returns the model API error if it is a quota or server error. + + The tool under test cannot cause these errors, so the trial is + unscored. Each CLI overrides this. By default, every error is a failure. + """ + return None diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index b4595a3b6..cb44f5123 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -945,6 +945,16 @@ def _parse_stream_json( # Assistant message with content blocks message = event.get("message", {}) content_blocks = message.get("content", []) + if event.get("error") and not event.get("parent_tool_use_id"): + # Set when the API call fails, e.g. "rate_limit". + # A subagent error is skipped, because the main + # agent receives it as a tool result and continues. + final_obj["api_error"] = { + "type": event["error"], + "message": "".join( + b.get("text", "") for b in content_blocks + if isinstance(b, dict)), + } for block in content_blocks: if not isinstance(block, dict): continue @@ -1166,6 +1176,16 @@ def _parse_stream_json( def version(self) -> str: return self.claude_code_version + def model_api_error(self, result: subprocess.CompletedProcess) -> Optional[str]: + """Returns the API error for "rate_limit" or "server_error" (any 5xx).""" + try: + api_error = json.loads(result.stdout or "{}").get("api_error") or {} + except (json.JSONDecodeError, AttributeError): + return None + if api_error.get("type") in ("rate_limit", "server_error"): + return api_error.get("message") or api_error["type"] + return None + def parse_response(self, stdout: str) -> dict: if not stdout: return {} diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index 5cb0af764..f4e1441d6 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -20,6 +20,14 @@ ) _SECRET_MANAGER_URL_PREFIX = "secret_manager://" +# Quota and server errors in a Codex `turn.failed` message. "at capacity" is +# CodexErr::ServerOverloaded and "high demand" is CodexErr::InternalServerError. +_CODEX_QUOTA_OR_SERVER_RE = re.compile( + r"\b(?:429|500|502|503|504)\b|quota|usage limit|rate limit" + r"|too many requests|stream disconnected before completion" + r"|at capacity|high demand", + re.IGNORECASE) + def _strip_sm_prefix(value: str) -> str: if value.startswith(_SECRET_MANAGER_URL_PREFIX): @@ -1014,6 +1022,13 @@ def item_payload(item: dict) -> dict: usage = event.get("usage", {}) or usage continue + if event_type == "turn.failed": + # Only the final error counts. `error` events are retries. + error = event.get("error") + final_obj["turn_error"] = str( + error.get("message", "") if isinstance(error, dict) else error or "") + continue + if event_type == "error": # surface the error text into the response field msg = event.get("message", "") @@ -1242,6 +1257,16 @@ def _coerce_json(value): def version(self) -> str: return self.codex_cli_version + def model_api_error(self, result: subprocess.CompletedProcess) -> Optional[str]: + """Returns the `turn.failed` message if it is a quota or server error.""" + try: + turn_error = json.loads(result.stdout or "{}").get("turn_error") or "" + except (json.JSONDecodeError, AttributeError): + return None + if _CODEX_QUOTA_OR_SERVER_RE.search(turn_error): + return turn_error + return None + def parse_response(self, stdout: str) -> dict: if not stdout: return {} diff --git a/evalbench/generators/models/gemini_cli.py b/evalbench/generators/models/gemini_cli.py index 37488cff2..f005ea185 100644 --- a/evalbench/generators/models/gemini_cli.py +++ b/evalbench/generators/models/gemini_cli.py @@ -11,6 +11,14 @@ from util.context import rpc_id_var from util.sessionmgr import validate_session_id +# Quota and server errors in a Gemini "[API Error: ...]" message. A bare +# "quota" does not match, because a missing quota project is a config error. +_GEMINI_QUOTA_OR_SERVER_RE = re.compile( + r"\b(?:429|500|502|503|504)\b|RESOURCE_EXHAUSTED|UNAVAILABLE|\bINTERNAL\b" + r"|DEADLINE_EXCEEDED|quota exceeded|exceeded[^\n]{0,40}quota|exhausted" + r"|rate limit|too many requests|overloaded", + re.IGNORECASE) + class CLICommand: def __init__(self, cli, prompt, env=None, resume=False, yolo=True, cwd=None, @@ -1044,6 +1052,28 @@ def _parse_stream_json(self, stream_output: str) -> str: def version(self) -> str: return self.gemini_cli_version + def model_api_error(self, result: subprocess.CompletedProcess) -> Optional[str]: + """Returns the final API error if it is a quota or server error. + + On failure, stdout is the raw stream-json. Gemini CLI formats fatal + model errors as "[API Error: ...]". Tool errors and retry logs use + other formats. + """ + for line in reversed((result.stdout or "").splitlines()): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict) or event.get("type") != "result": + continue + if event.get("status") != "error": + return None + message = str((event.get("error") or {}).get("message", "")) + if message.startswith("[API Error:") and _GEMINI_QUOTA_OR_SERVER_RE.search(message): + return message + return None + return None + def parse_response(self, stdout: str) -> dict: if not stdout: return {} diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index b047b248c..fc5d9d2d9 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -220,8 +220,7 @@ def analyze_one_metric( "skills_best_practices", ] if metric_name in non_binary_metrics: - # Scorers return 0 for a trial with a generated_error, such as - # an agent quota error. Leave those rows out of the average. + # Error rows score 0. Leave them out of the average. if "generated_error" in df_metric.columns: df_metric = df_metric[ df_metric["generated_error"].isna() diff --git a/evalbench/test/agent_infra_errors_test.py b/evalbench/test/agent_infra_errors_test.py index dfa04ab50..04db47c67 100644 --- a/evalbench/test/agent_infra_errors_test.py +++ b/evalbench/test/agent_infra_errors_test.py @@ -1,35 +1,147 @@ +import json import subprocess from unittest.mock import MagicMock, patch import pandas as pd import pytest -from evaluator.agentevaluator import AgentEvaluator, _infra_error +from evaluator.agentevaluator import AgentEvaluator from generators.models.agent_cli import AgentCliGenerator +from generators.models.claude_code import ClaudeCodeGenerator +from generators.models.codex_cli import CodexCliGenerator +from generators.models.gemini_cli import GeminiCliGenerator +from generators.models.noop_agent import NoopAgentGenerator +from generators.models.passthrough import NOOPGenerator from reporting.analyzer import analyze_one_metric from work.agentscorework import AgentScoreWork -def _result(returncode, stderr=""): - return subprocess.CompletedProcess([], returncode, "agent output", stderr) +def _result(returncode, stderr="", stdout="agent output"): + return subprocess.CompletedProcess([], returncode, stdout, stderr) -@pytest.mark.parametrize("returncode, stderr, is_infra", [ - (1, "Error: 429 Too Many Requests", True), - (1, "status: RESOURCE_EXHAUSTED", True), - (1, "HTTP 503 Service Unavailable", True), - (1, "API status code 502", True), - (1, "Quota exceeded for aiplatform requests", True), - (124, "TimeoutError after 503 seconds", False), - (0, "Error 429", False), - (1, "Tool failed: table not found", False), - (1, 'File "tool.py", line 503, in run', False), - (1, "WARNING: credentials without a quota project", False), -], ids=["429", "resource_exhausted", "503", "status_code_502", - "quota_exceeded", "timeout", "success", "agent_failure", - "traceback_line_number", "quota_project_warning"]) -def test_infra_error_detection(returncode, stderr, is_infra): - assert bool(_infra_error(_result(returncode, stderr))) == is_infra +def _stream(*events): + return "\n".join(json.dumps(e) for e in events) + + +def _bare(cls): + """Returns a generator without running __init__, which sets up a sandbox.""" + generator = object.__new__(cls) + generator.model = "test-model" + generator.pricing = None + return generator + + +def _gemini_result(status, message=None, stderr=""): + result_event = {"type": "result", "status": status, "stats": {}} + if message is not None: + result_event["error"] = {"type": "Error", "message": message} + stdout = _stream({"type": "init", "session_id": "s"}, result_event) + return _result(0 if status == "success" else 1, stderr, stdout) + + +@pytest.mark.parametrize("result, is_infra", [ + (_gemini_result("error", '[API Error: {"error":{"code":429,"status":' + '"RESOURCE_EXHAUSTED"}}]\nPlease wait and try again later.'), + True), + (_gemini_result("error", "[API Error: got status: 503 Service Unavailable]"), + True), + (_gemini_result("error", "[API Error: You have exhausted your daily quota.]"), + True), + (_gemini_result("error", "[API Error: Deadline expired before operation " + "could complete. (Status: DEADLINE_EXCEEDED)]"), True), + (_gemini_result("error", "Error executing tool bigquery__execute_sql: " + "HTTP 500 UNAVAILABLE"), False), + (_gemini_result("error", "[API Error: Request contains an invalid argument. " + "(Status: INVALID_ARGUMENT)]"), False), + (_gemini_result("error", "[API Error: The aiplatform.googleapis.com API " + "requires a quota project, which is not set by default. " + "(Status: PERMISSION_DENIED)]"), False), + (_gemini_result("success", stderr="Attempt 1 failed with status 429. " + "Retrying with backoff..."), False), +], ids=["quota_429", "server_503", "daily_quota", "deadline_exceeded", + "tool_error", "invalid_argument", "quota_project_missing", + "retry_log_then_success"]) +def test_gemini_model_api_error(result, is_infra): + error = _bare(GeminiCliGenerator).model_api_error(result) + assert bool(error) == is_infra + + +def _claude_result(*events): + generator = _bare(ClaudeCodeGenerator) + return generator, _result(1, "", generator._parse_stream_json(_stream(*events))) + + +def _claude_api_error(category, text): + return {"type": "assistant", "error": category, + "message": {"content": [{"type": "text", "text": text}]}} + + +@pytest.mark.parametrize("events, is_infra", [ + ([_claude_api_error("server_error", 'API Error: 529 {"type":"error","error":' + '{"type":"overloaded_error","message":"Overloaded"}}')], + True), + ([_claude_api_error("rate_limit", "API Error: Request rejected (429)")], True), + ([_claude_api_error("authentication_failed", "Invalid API key")], False), + ([{"type": "user", "message": {"content": [{ + "type": "tool_result", "tool_use_id": "t1", "is_error": True, + "content": "cloud-sql__list_instances returned UNAVAILABLE"}]}}], False), +], ids=["overloaded_529", "rate_limit", "auth_failed", "tool_error_on_stdout"]) +def test_claude_model_api_error(events, is_infra): + generator, result = _claude_result(*events) + assert bool(generator.model_api_error(result)) == is_infra + + +def test_claude_subagent_api_error_is_ignored(): + """The main agent gets a subagent API error as a tool result and continues.""" + subagent_error = _claude_api_error("rate_limit", + "API Error: Request rejected (429)") + subagent_error["parent_tool_use_id"] = "toolu_task" + generator, result = _claude_result( + subagent_error, + {"type": "assistant", "parent_tool_use_id": None, + "message": {"content": [{"type": "text", "text": "Done."}]}}) + assert generator.model_api_error(result) is None + + +def _codex_result(*events): + generator = _bare(CodexCliGenerator) + return generator, _result(1, "", generator._parse_stream_json(_stream(*events))) + + +@pytest.mark.parametrize("events, is_infra", [ + ([{"type": "error", "message": "stream disconnected - retrying sampling " + "request (1/5)"}, + {"type": "turn.failed", "error": {"message": "exceeded retry limit, " + "last status: 429 Too Many Requests"}}], + True), + ([{"type": "turn.failed", "error": {"message": "unexpected status 503 " + "Service Unavailable"}}], True), + ([{"type": "turn.failed", "error": {"message": "Quota exceeded. Check your " + "plan and billing details."}}], True), + ([{"type": "turn.failed", "error": {"message": "Selected model is at " + "capacity. Please try a different " + "model."}}], True), + ([{"type": "turn.failed", "error": {"message": "We're currently " + "experiencing high demand, which may " + "cause temporary errors."}}], True), + ([{"type": "error", "message": "stream disconnected - retrying sampling " + "request (1/5)"}, + {"type": "turn.completed", "usage": {}}], False), + ([{"type": "turn.failed", "error": {"message": "context window exceeded"}}], + False), +], ids=["retry_limit_429", "unexpected_503", "quota", "server_overloaded", + "internal_server_error", "retry_then_success", "context_window"]) +def test_codex_model_api_error(events, is_infra): + generator, result = _codex_result(*events) + assert bool(generator.model_api_error(result)) == is_infra + + +def test_default_model_api_error_ignores_stderr(): + """A gRPC proxy error_message comes from the remote agent and its tools.""" + result = _result(1, "Agent reported error: dataform__compile failed with " + "code 503 UNAVAILABLE") + assert AgentCliGenerator.model_api_error(MagicMock(), result) is None @pytest.fixture @@ -43,11 +155,15 @@ def run_scenario(): generator.parse_response.return_value = {} generator.extract_tools.return_value = [] generator.extract_skills.return_value = [] + generator.model_api_error.return_value = None patcher.start().return_value = generator - def _run(cli_result, simulated_user_reply): + def _run(cli_result, simulated_user_reply, model_generator=None, + model_api_error=None): generator.safe_generate.return_value = cli_result + generator.model_api_error.return_value = model_api_error simulated_user = MagicMock() + simulated_user.model_generator = model_generator or MagicMock() if isinstance(simulated_user_reply, Exception): simulated_user.get_next_response.side_effect = simulated_user_reply else: @@ -65,20 +181,42 @@ def _run(cli_result, simulated_user_reply): def test_infra_error_stops_the_conversation(run_scenario): - error, simulated_user = run_scenario(_result(1, "429 Too Many Requests"), - "next") - assert error.startswith("Agent infrastructure error (exit 1)") + error, simulated_user = run_scenario( + _result(1), "next", model_api_error="429 Too Many Requests") + assert error == "Agent infrastructure error (exit 1): 429 Too Many Requests" simulated_user.get_next_response.assert_not_called() +def test_timeout_is_not_an_infra_error(run_scenario): + error, _ = run_scenario(_result(124), "TERMINATE", + model_api_error="503 Service Unavailable") + assert error is None + + def test_simulated_user_error_is_recorded(run_scenario): error, _ = run_scenario(_result(0), RuntimeError("model down")) assert error == "Simulated user error: model down" -def test_empty_simulated_user_reply_continues(run_scenario): +@pytest.mark.parametrize("reply", ["", None], ids=["empty", "none"]) +def test_empty_simulated_user_reply_is_recorded(run_scenario, reply): + """QueryGenerator.generate returns "" after a simulated-user quota error. + + ClaudeGenerator returns None on any error. + """ + error, simulated_user = run_scenario(_result(0), reply) + assert error == ("Simulated user error: " + "empty reply from the simulated user model") + assert simulated_user.get_next_response.call_count == 1 + + +@pytest.mark.parametrize("noop_generator", [ + MagicMock(spec=NOOPGenerator), MagicMock(spec=NoopAgentGenerator), +], ids=["noop", "noop_agent"]) +def test_empty_noop_simulated_user_reply_continues(run_scenario, + noop_generator): """The noop simulated user of proxy flows returns "" by design.""" - error, simulated_user = run_scenario(_result(0), "") + error, simulated_user = run_scenario(_result(0), "", noop_generator) assert error is None assert simulated_user.get_next_response.call_count == 2 diff --git a/evalbench/test/agentevaluator_test.py b/evalbench/test/agentevaluator_test.py index de3176dc4..e8e18c1f0 100644 --- a/evalbench/test/agentevaluator_test.py +++ b/evalbench/test/agentevaluator_test.py @@ -36,6 +36,7 @@ def tearDown(self): def test_process_scenario_copies_env_files(self, mock_get_generator): # 1. Setup mocks mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None # Set fake_home on the generator to point to our temp fake_home mock_generator.fake_home = self.fake_home mock_generator.name = "mock_agent_cli" diff --git a/evalbench/test/test_eval_case_timeout.py b/evalbench/test/test_eval_case_timeout.py index 2e4d01baa..984bb1ef7 100644 --- a/evalbench/test/test_eval_case_timeout.py +++ b/evalbench/test/test_eval_case_timeout.py @@ -77,6 +77,7 @@ class TestAgentEvaluatorTimeout(unittest.TestCase): @patch("evaluator.agentevaluator.get_generator") def test_agent_evaluator_initializes_timeout(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_get_generator.return_value = mock_generator @@ -99,6 +100,7 @@ def test_agent_evaluator_initializes_timeout(self, mock_get_generator): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_times_out_multi_turn(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" @@ -154,6 +156,7 @@ def slow_safe_generate(cli_cmd, timeout_seconds=None): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_times_out_before_turn_1(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_get_generator.return_value = mock_generator @@ -189,6 +192,7 @@ def test_process_scenario_times_out_before_turn_1(self, mock_get_generator): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_scenario_level_timeout_override(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" @@ -236,6 +240,7 @@ def slow_safe_generate(cli_cmd, timeout_seconds=None): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_records_timeout_error_in_result(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" @@ -309,6 +314,7 @@ def test_agent_score_work_keeps_generated_error_none(self): @patch("evaluator.agentevaluator.get_generator") def test_evaluate_agent_cli_returns_job_id_and_outputs_on_timeout(self, mock_get_generator, mock_sim_user): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home"