diff --git a/evalbench/evaluator/agentevaluator.py b/evalbench/evaluator/agentevaluator.py index e81fd5562..5dd6a3320 100644 --- a/evalbench/evaluator/agentevaluator.py +++ b/evalbench/evaluator/agentevaluator.py @@ -10,6 +10,8 @@ from dataset.evalgeminicliinput import EvalGeminiCliRequest from generators.models import get_generator from generators.models.agent_cli import AgentCliGenerator +from generators.models.noop_agent import NoopAgentGenerator +from generators.models.passthrough import NOOPGenerator from mp import mprunner from util.config import get_eval_case_timeout from work.agentgenwork import AgentGenWork @@ -20,6 +22,24 @@ from typing import Dict +def _infra_error(generator: AgentCliGenerator, result: Any) -> str | None: + """Returns an error message if the model API hit a quota or server error.""" + returncode = getattr(result, "returncode", 0) + # 124 is a timeout, which counts as a failure. + if returncode == 124: + return None + api_error = generator.model_api_error(result) + if not api_error: + return None + return f"Agent infrastructure error (exit {returncode}): {api_error.strip()[-500:]}" + + +def _is_noop(simulated_user: SimulatedUser) -> bool: + """Returns True if the simulated user returns an empty reply by design.""" + return isinstance(simulated_user.model_generator, + (NOOPGenerator, NoopAgentGenerator)) + + class AgentEvaluator: def __init__( self, @@ -162,6 +182,7 @@ def process_scenario( else self.eval_case_timeout_seconds ) last_result = None + generated_error = None for turn in range(max_turns): remaining_timeout_seconds = None @@ -241,13 +262,29 @@ def process_scenario( "agent": result.stdout }) + generated_error = _infra_error(self.generator, result) + if generated_error: + logging.error(generated_error) + break + if turn < max_turns - 1: if simulated_user: - next_response = simulated_user.get_next_response( - conversation_plan, - conversation_history, - result.stdout - ) + try: + next_response = simulated_user.get_next_response( + conversation_plan, + conversation_history, + result.stdout + ) + # QueryGenerator.generate returns "" when out of quota. + # ClaudeGenerator returns None on any error. + if (not str(next_response or "").strip() + and not _is_noop(simulated_user)): + raise RuntimeError( + "empty reply from the simulated user model") + except Exception as e: + generated_error = f"Simulated user error: {e}" + logging.error(generated_error) + break if "TERMINATE" in next_response: logging.info("Simulated user terminated conversation.") break @@ -268,7 +305,8 @@ def process_scenario( accumulated_skills, eval_result, job_id, - metadata + metadata, + generated_error, ) def _log_cli_result(self, turn: int, max_turns: int, result: subprocess.CompletedProcess): @@ -289,7 +327,8 @@ def _finalize_scenario( accumulated_skills: List[str], eval_result: Any, job_id: str, - metadata: Dict[str, Any] + metadata: Dict[str, Any], + generated_error: str | None = None, ): """Finalizes the scenario by scoring and appending results.""" timed_out = ( @@ -305,7 +344,7 @@ def _finalize_scenario( "returncode": getattr(last_result, "returncode", 0), "timed_out": timed_out, "prompt_generator_error": None, - "generated_error": None, + "generated_error": generated_error, "sql_generator_error": None, "golden_error": None, "generated_sql": "skipped", diff --git a/evalbench/generators/models/agent_cli.py b/evalbench/generators/models/agent_cli.py index e8d9e2e94..3f167087b 100644 --- a/evalbench/generators/models/agent_cli.py +++ b/evalbench/generators/models/agent_cli.py @@ -114,3 +114,13 @@ def extract_tools(self, stdout: str) -> list: @abstractmethod def extract_skills(self, stdout: str) -> list: raise NotImplementedError("Subclasses must implement this method") + + def model_api_error( + self, result: subprocess.CompletedProcess + ) -> Optional[str]: + """Returns the model API error if it is a quota or server error. + + The tool under test cannot cause these errors, so the trial is + unscored. Each CLI overrides this. By default, every error is a failure. + """ + return None diff --git a/evalbench/generators/models/claude_code.py b/evalbench/generators/models/claude_code.py index 3456ad136..1829b7d2e 100644 --- a/evalbench/generators/models/claude_code.py +++ b/evalbench/generators/models/claude_code.py @@ -947,6 +947,16 @@ def _parse_stream_json( # Assistant message with content blocks message = event.get("message", {}) content_blocks = message.get("content", []) + if event.get("error") and not event.get("parent_tool_use_id"): + # Set when the API call fails, e.g. "rate_limit". + # A subagent error is skipped, because the main + # agent receives it as a tool result and continues. + final_obj["api_error"] = { + "type": event["error"], + "message": "".join( + b.get("text", "") for b in content_blocks + if isinstance(b, dict)), + } for block in content_blocks: if not isinstance(block, dict): continue @@ -1168,6 +1178,16 @@ def _parse_stream_json( def version(self) -> str: return self.claude_code_version + def model_api_error(self, result: subprocess.CompletedProcess) -> Optional[str]: + """Returns the API error for "rate_limit" or "server_error" (any 5xx).""" + try: + api_error = json.loads(result.stdout or "{}").get("api_error") or {} + except (json.JSONDecodeError, AttributeError): + return None + if api_error.get("type") in ("rate_limit", "server_error"): + return api_error.get("message") or api_error["type"] + return None + def parse_response(self, stdout: str) -> dict: if not stdout: return {} diff --git a/evalbench/generators/models/codex_cli.py b/evalbench/generators/models/codex_cli.py index 5cb0af764..f4e1441d6 100644 --- a/evalbench/generators/models/codex_cli.py +++ b/evalbench/generators/models/codex_cli.py @@ -20,6 +20,14 @@ ) _SECRET_MANAGER_URL_PREFIX = "secret_manager://" +# Quota and server errors in a Codex `turn.failed` message. "at capacity" is +# CodexErr::ServerOverloaded and "high demand" is CodexErr::InternalServerError. +_CODEX_QUOTA_OR_SERVER_RE = re.compile( + r"\b(?:429|500|502|503|504)\b|quota|usage limit|rate limit" + r"|too many requests|stream disconnected before completion" + r"|at capacity|high demand", + re.IGNORECASE) + def _strip_sm_prefix(value: str) -> str: if value.startswith(_SECRET_MANAGER_URL_PREFIX): @@ -1014,6 +1022,13 @@ def item_payload(item: dict) -> dict: usage = event.get("usage", {}) or usage continue + if event_type == "turn.failed": + # Only the final error counts. `error` events are retries. + error = event.get("error") + final_obj["turn_error"] = str( + error.get("message", "") if isinstance(error, dict) else error or "") + continue + if event_type == "error": # surface the error text into the response field msg = event.get("message", "") @@ -1242,6 +1257,16 @@ def _coerce_json(value): def version(self) -> str: return self.codex_cli_version + def model_api_error(self, result: subprocess.CompletedProcess) -> Optional[str]: + """Returns the `turn.failed` message if it is a quota or server error.""" + try: + turn_error = json.loads(result.stdout or "{}").get("turn_error") or "" + except (json.JSONDecodeError, AttributeError): + return None + if _CODEX_QUOTA_OR_SERVER_RE.search(turn_error): + return turn_error + return None + def parse_response(self, stdout: str) -> dict: if not stdout: return {} diff --git a/evalbench/generators/models/gemini_cli.py b/evalbench/generators/models/gemini_cli.py index 37488cff2..f005ea185 100644 --- a/evalbench/generators/models/gemini_cli.py +++ b/evalbench/generators/models/gemini_cli.py @@ -11,6 +11,14 @@ from util.context import rpc_id_var from util.sessionmgr import validate_session_id +# Quota and server errors in a Gemini "[API Error: ...]" message. A bare +# "quota" does not match, because a missing quota project is a config error. +_GEMINI_QUOTA_OR_SERVER_RE = re.compile( + r"\b(?:429|500|502|503|504)\b|RESOURCE_EXHAUSTED|UNAVAILABLE|\bINTERNAL\b" + r"|DEADLINE_EXCEEDED|quota exceeded|exceeded[^\n]{0,40}quota|exhausted" + r"|rate limit|too many requests|overloaded", + re.IGNORECASE) + class CLICommand: def __init__(self, cli, prompt, env=None, resume=False, yolo=True, cwd=None, @@ -1044,6 +1052,28 @@ def _parse_stream_json(self, stream_output: str) -> str: def version(self) -> str: return self.gemini_cli_version + def model_api_error(self, result: subprocess.CompletedProcess) -> Optional[str]: + """Returns the final API error if it is a quota or server error. + + On failure, stdout is the raw stream-json. Gemini CLI formats fatal + model errors as "[API Error: ...]". Tool errors and retry logs use + other formats. + """ + for line in reversed((result.stdout or "").splitlines()): + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict) or event.get("type") != "result": + continue + if event.get("status") != "error": + return None + message = str((event.get("error") or {}).get("message", "")) + if message.startswith("[API Error:") and _GEMINI_QUOTA_OR_SERVER_RE.search(message): + return message + return None + return None + def parse_response(self, stdout: str) -> dict: if not stdout: return {} diff --git a/evalbench/reporting/analyzer.py b/evalbench/reporting/analyzer.py index 6068355ad..fc5d9d2d9 100644 --- a/evalbench/reporting/analyzer.py +++ b/evalbench/reporting/analyzer.py @@ -220,6 +220,12 @@ def analyze_one_metric( "skills_best_practices", ] if metric_name in non_binary_metrics: + # Error rows score 0. Leave them out of the average. + if "generated_error" in df_metric.columns: + df_metric = df_metric[ + df_metric["generated_error"].isna() + | (df_metric["generated_error"] == "") + ] avg_val = ( df_metric["score"].mean() if (not df_metric.empty and "score" in df_metric.columns) diff --git a/evalbench/test/agent_infra_errors_test.py b/evalbench/test/agent_infra_errors_test.py new file mode 100644 index 000000000..04db47c67 --- /dev/null +++ b/evalbench/test/agent_infra_errors_test.py @@ -0,0 +1,260 @@ +import json +import subprocess +from unittest.mock import MagicMock, patch + +import pandas as pd +import pytest + +from evaluator.agentevaluator import AgentEvaluator +from generators.models.agent_cli import AgentCliGenerator +from generators.models.claude_code import ClaudeCodeGenerator +from generators.models.codex_cli import CodexCliGenerator +from generators.models.gemini_cli import GeminiCliGenerator +from generators.models.noop_agent import NoopAgentGenerator +from generators.models.passthrough import NOOPGenerator +from reporting.analyzer import analyze_one_metric +from work.agentscorework import AgentScoreWork + + +def _result(returncode, stderr="", stdout="agent output"): + return subprocess.CompletedProcess([], returncode, stdout, stderr) + + +def _stream(*events): + return "\n".join(json.dumps(e) for e in events) + + +def _bare(cls): + """Returns a generator without running __init__, which sets up a sandbox.""" + generator = object.__new__(cls) + generator.model = "test-model" + generator.pricing = None + return generator + + +def _gemini_result(status, message=None, stderr=""): + result_event = {"type": "result", "status": status, "stats": {}} + if message is not None: + result_event["error"] = {"type": "Error", "message": message} + stdout = _stream({"type": "init", "session_id": "s"}, result_event) + return _result(0 if status == "success" else 1, stderr, stdout) + + +@pytest.mark.parametrize("result, is_infra", [ + (_gemini_result("error", '[API Error: {"error":{"code":429,"status":' + '"RESOURCE_EXHAUSTED"}}]\nPlease wait and try again later.'), + True), + (_gemini_result("error", "[API Error: got status: 503 Service Unavailable]"), + True), + (_gemini_result("error", "[API Error: You have exhausted your daily quota.]"), + True), + (_gemini_result("error", "[API Error: Deadline expired before operation " + "could complete. (Status: DEADLINE_EXCEEDED)]"), True), + (_gemini_result("error", "Error executing tool bigquery__execute_sql: " + "HTTP 500 UNAVAILABLE"), False), + (_gemini_result("error", "[API Error: Request contains an invalid argument. " + "(Status: INVALID_ARGUMENT)]"), False), + (_gemini_result("error", "[API Error: The aiplatform.googleapis.com API " + "requires a quota project, which is not set by default. " + "(Status: PERMISSION_DENIED)]"), False), + (_gemini_result("success", stderr="Attempt 1 failed with status 429. " + "Retrying with backoff..."), False), +], ids=["quota_429", "server_503", "daily_quota", "deadline_exceeded", + "tool_error", "invalid_argument", "quota_project_missing", + "retry_log_then_success"]) +def test_gemini_model_api_error(result, is_infra): + error = _bare(GeminiCliGenerator).model_api_error(result) + assert bool(error) == is_infra + + +def _claude_result(*events): + generator = _bare(ClaudeCodeGenerator) + return generator, _result(1, "", generator._parse_stream_json(_stream(*events))) + + +def _claude_api_error(category, text): + return {"type": "assistant", "error": category, + "message": {"content": [{"type": "text", "text": text}]}} + + +@pytest.mark.parametrize("events, is_infra", [ + ([_claude_api_error("server_error", 'API Error: 529 {"type":"error","error":' + '{"type":"overloaded_error","message":"Overloaded"}}')], + True), + ([_claude_api_error("rate_limit", "API Error: Request rejected (429)")], True), + ([_claude_api_error("authentication_failed", "Invalid API key")], False), + ([{"type": "user", "message": {"content": [{ + "type": "tool_result", "tool_use_id": "t1", "is_error": True, + "content": "cloud-sql__list_instances returned UNAVAILABLE"}]}}], False), +], ids=["overloaded_529", "rate_limit", "auth_failed", "tool_error_on_stdout"]) +def test_claude_model_api_error(events, is_infra): + generator, result = _claude_result(*events) + assert bool(generator.model_api_error(result)) == is_infra + + +def test_claude_subagent_api_error_is_ignored(): + """The main agent gets a subagent API error as a tool result and continues.""" + subagent_error = _claude_api_error("rate_limit", + "API Error: Request rejected (429)") + subagent_error["parent_tool_use_id"] = "toolu_task" + generator, result = _claude_result( + subagent_error, + {"type": "assistant", "parent_tool_use_id": None, + "message": {"content": [{"type": "text", "text": "Done."}]}}) + assert generator.model_api_error(result) is None + + +def _codex_result(*events): + generator = _bare(CodexCliGenerator) + return generator, _result(1, "", generator._parse_stream_json(_stream(*events))) + + +@pytest.mark.parametrize("events, is_infra", [ + ([{"type": "error", "message": "stream disconnected - retrying sampling " + "request (1/5)"}, + {"type": "turn.failed", "error": {"message": "exceeded retry limit, " + "last status: 429 Too Many Requests"}}], + True), + ([{"type": "turn.failed", "error": {"message": "unexpected status 503 " + "Service Unavailable"}}], True), + ([{"type": "turn.failed", "error": {"message": "Quota exceeded. Check your " + "plan and billing details."}}], True), + ([{"type": "turn.failed", "error": {"message": "Selected model is at " + "capacity. Please try a different " + "model."}}], True), + ([{"type": "turn.failed", "error": {"message": "We're currently " + "experiencing high demand, which may " + "cause temporary errors."}}], True), + ([{"type": "error", "message": "stream disconnected - retrying sampling " + "request (1/5)"}, + {"type": "turn.completed", "usage": {}}], False), + ([{"type": "turn.failed", "error": {"message": "context window exceeded"}}], + False), +], ids=["retry_limit_429", "unexpected_503", "quota", "server_overloaded", + "internal_server_error", "retry_then_success", "context_window"]) +def test_codex_model_api_error(events, is_infra): + generator, result = _codex_result(*events) + assert bool(generator.model_api_error(result)) == is_infra + + +def test_default_model_api_error_ignores_stderr(): + """A gRPC proxy error_message comes from the remote agent and its tools.""" + result = _result(1, "Agent reported error: dataform__compile failed with " + "code 503 UNAVAILABLE") + assert AgentCliGenerator.model_api_error(MagicMock(), result) is None + + +@pytest.fixture +def run_scenario(): + """Runs one 3-turn scenario and returns (generated_error, simulated_user).""" + patcher = patch("evaluator.agentevaluator.get_generator") + generator = MagicMock(spec=AgentCliGenerator) + generator.version = "1.0" + generator.name = "mock_cli" + generator.fake_home = None + generator.parse_response.return_value = {} + generator.extract_tools.return_value = [] + generator.extract_skills.return_value = [] + generator.model_api_error.return_value = None + patcher.start().return_value = generator + + def _run(cli_result, simulated_user_reply, model_generator=None, + model_api_error=None): + generator.safe_generate.return_value = cli_result + generator.model_api_error.return_value = model_api_error + simulated_user = MagicMock() + simulated_user.model_generator = model_generator or MagicMock() + if isinstance(simulated_user_reply, Exception): + simulated_user.get_next_response.side_effect = simulated_user_reply + else: + simulated_user.get_next_response.return_value = simulated_user_reply + evaluator = AgentEvaluator({"model_config": "dummy.yaml"}) + evaluator._finalize_scenario = MagicMock() + evaluator.process_scenario( + {"id": "s1", "starting_prompt": "hi", "max_turns": 3}, + eval_result=MagicMock(), job_id="job1", metadata={}, + simulated_user=simulated_user) + return evaluator._finalize_scenario.call_args.args[-1], simulated_user + + yield _run + patcher.stop() + + +def test_infra_error_stops_the_conversation(run_scenario): + error, simulated_user = run_scenario( + _result(1), "next", model_api_error="429 Too Many Requests") + assert error == "Agent infrastructure error (exit 1): 429 Too Many Requests" + simulated_user.get_next_response.assert_not_called() + + +def test_timeout_is_not_an_infra_error(run_scenario): + error, _ = run_scenario(_result(124), "TERMINATE", + model_api_error="503 Service Unavailable") + assert error is None + + +def test_simulated_user_error_is_recorded(run_scenario): + error, _ = run_scenario(_result(0), RuntimeError("model down")) + assert error == "Simulated user error: model down" + + +@pytest.mark.parametrize("reply", ["", None], ids=["empty", "none"]) +def test_empty_simulated_user_reply_is_recorded(run_scenario, reply): + """QueryGenerator.generate returns "" after a simulated-user quota error. + + ClaudeGenerator returns None on any error. + """ + error, simulated_user = run_scenario(_result(0), reply) + assert error == ("Simulated user error: " + "empty reply from the simulated user model") + assert simulated_user.get_next_response.call_count == 1 + + +@pytest.mark.parametrize("noop_generator", [ + MagicMock(spec=NOOPGenerator), MagicMock(spec=NoopAgentGenerator), +], ids=["noop", "noop_agent"]) +def test_empty_noop_simulated_user_reply_continues(run_scenario, + noop_generator): + """The noop simulated user of proxy flows returns "" by design.""" + error, simulated_user = run_scenario(_result(0), "", noop_generator) + assert error is None + assert simulated_user.get_next_response.call_count == 2 + + +def test_normal_run_has_no_error(run_scenario): + error, _ = run_scenario(_result(0), "TERMINATE") + assert error is None + + +def test_score_work_passes_generated_error(): + eval_output = {"eval_id": "s1", "generated_error": "Agent infrastructure error", + "scenario": {}, "metadata": {}} + with patch("scorers.score.compare") as mock_compare: + AgentScoreWork(config={}, eval_output=eval_output, + scoring_results=[]).run() + item = mock_compare.call_args.kwargs["eval_output_item"] + assert item["generated_error"] == "Agent infrastructure error" + + +@pytest.mark.parametrize("generated_error", ["Agent infrastructure error", None]) +def test_finalize_scenario_writes_generated_error(generated_error): + with patch("evaluator.agentevaluator.get_generator") as mock_get_generator: + mock_get_generator.return_value = MagicMock(spec=AgentCliGenerator) + evaluator = AgentEvaluator({"model_config": "dummy.yaml"}) + eval_result = MagicMock(agent_results=[], scoring_results=[]) + with patch("evaluator.agentevaluator.AgentScoreWork"): + evaluator._finalize_scenario( + {"id": "s1", "starting_prompt": "hi"}, _result(1), [], [], [], + eval_result, "job1", {}, generated_error) + assert eval_result.agent_results[0]["generated_error"] == generated_error + + +def test_average_skips_rows_with_generated_error(): + df = pd.DataFrame([ + {"comparator": "end_to_end_latency", "score": 1000.0, + "generated_error": None}, + {"comparator": "end_to_end_latency", "score": 0.0, + "generated_error": "Agent infrastructure error"}, + ]) + summary = analyze_one_metric(df, "end_to_end_latency", 100) + assert summary["metric_score"] == 1000.0 diff --git a/evalbench/test/agentevaluator_test.py b/evalbench/test/agentevaluator_test.py index de3176dc4..e8e18c1f0 100644 --- a/evalbench/test/agentevaluator_test.py +++ b/evalbench/test/agentevaluator_test.py @@ -36,6 +36,7 @@ def tearDown(self): def test_process_scenario_copies_env_files(self, mock_get_generator): # 1. Setup mocks mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None # Set fake_home on the generator to point to our temp fake_home mock_generator.fake_home = self.fake_home mock_generator.name = "mock_agent_cli" diff --git a/evalbench/test/test_eval_case_timeout.py b/evalbench/test/test_eval_case_timeout.py index 2e4d01baa..984bb1ef7 100644 --- a/evalbench/test/test_eval_case_timeout.py +++ b/evalbench/test/test_eval_case_timeout.py @@ -77,6 +77,7 @@ class TestAgentEvaluatorTimeout(unittest.TestCase): @patch("evaluator.agentevaluator.get_generator") def test_agent_evaluator_initializes_timeout(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_get_generator.return_value = mock_generator @@ -99,6 +100,7 @@ def test_agent_evaluator_initializes_timeout(self, mock_get_generator): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_times_out_multi_turn(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" @@ -154,6 +156,7 @@ def slow_safe_generate(cli_cmd, timeout_seconds=None): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_times_out_before_turn_1(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_get_generator.return_value = mock_generator @@ -189,6 +192,7 @@ def test_process_scenario_times_out_before_turn_1(self, mock_get_generator): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_scenario_level_timeout_override(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" @@ -236,6 +240,7 @@ def slow_safe_generate(cli_cmd, timeout_seconds=None): @patch("evaluator.agentevaluator.get_generator") def test_process_scenario_records_timeout_error_in_result(self, mock_get_generator): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" @@ -309,6 +314,7 @@ def test_agent_score_work_keeps_generated_error_none(self): @patch("evaluator.agentevaluator.get_generator") def test_evaluate_agent_cli_returns_job_id_and_outputs_on_timeout(self, mock_get_generator, mock_sim_user): mock_generator = MagicMock(spec=AgentCliGenerator) + mock_generator.model_api_error.return_value = None mock_generator.name = "mock_agent_cli" mock_generator.version = "1.0" mock_generator.fake_home = "/tmp/fake_home" diff --git a/evalbench/work/agentscorework.py b/evalbench/work/agentscorework.py index d3ec0d44a..6ffb69477 100644 --- a/evalbench/work/agentscorework.py +++ b/evalbench/work/agentscorework.py @@ -51,7 +51,7 @@ def run(self, work_config: Any = None) -> Any: "generated_sql": "skipped", "generated_result": self.eval_output.get("accumulated_tools", []), "eval_results": self.eval_output, - "generated_error": None, + "generated_error": self.eval_output.get("generated_error"), "dialects": metadata.get("dialects", []), "database": metadata.get("database", "unknown"), "job_id": self.eval_output.get("job_id"),