diff --git a/benchmarks/harbor/QUICKSTART.md b/benchmarks/harbor/QUICKSTART.md index 82ca8ab39..8d430e580 100644 --- a/benchmarks/harbor/QUICKSTART.md +++ b/benchmarks/harbor/QUICKSTART.md @@ -118,8 +118,10 @@ base. `AOB_RUNTIME_IMAGE` is unset in this shell and there is no local `assetopsbench/runtime:dev`, or it names an image that is not local. Redo step 2 in this shell. -**`unknown scorer 'llm_judge'`**: you are calling `evaluate` by hand without -`--scorer-default`. Copy what the task's `tests/test.sh` does. +**`unknown scorer 'llm_judge'`**: a scenario's `scenario_meta.json` routes it +to `llm_judge`, or `--scorer-default llm_judge` was passed, and no judge model +was given. Set `AOB_JUDGE_MODEL`, or pass `--judge-model` when calling +`evaluate` by hand. **The verifier scores 0 but the agent clearly answered**: check `verifier/test-stderr.txt`. The evaluator joins records to scenarios on diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index ba8b17e09..82b647984 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -307,13 +307,13 @@ on `main`, and a value above what Docker has fails the run ## Scoring `evaluation.loader` takes each scenario's `scoring_method` from -`scenario_meta.json`, defaulting to `static_json`, and it wins over -`--scorer-default`. The open profile and mini (whose FMEA scenarios use `fmea`) -need no judge model. `tests/test.sh` passes `--judge-model` only when -`AOB_JUDGE_MODEL` is set, and always passes `--scorer-default static_json`, -because the CLI's own default, `llm_judge`, is registered only with a judge -model and would otherwise fail every scenario with -`unknown scorer 'llm_judge'`. +`scenario_meta.json`, and it wins over `--scorer-default`. A scenario without +one falls back to `--scorer-default`, which is `static_json`. The open profile +and mini (whose FMEA scenarios use `fmea`) need no judge model. +`tests/test.sh` passes `--judge-model` only when `AOB_JUDGE_MODEL` is set, and +passes `--scorer-default static_json` to keep the fallback explicit. A scenario +whose `scenario_meta.json` asks for `llm_judge` stops the evaluation with +`unknown scorer 'llm_judge'` unless a judge model is given. The generator copies every file the loader reads into `tests/scenarios/scenario_N/`: `question.txt`, `groundtruth.txt`, diff --git a/benchmarks/harbor/template/tests/test.sh b/benchmarks/harbor/template/tests/test.sh index 7c07ca121..b2aca74a6 100755 --- a/benchmarks/harbor/template/tests/test.sh +++ b/benchmarks/harbor/template/tests/test.sh @@ -18,8 +18,8 @@ cd /opt/aob || { exit 0 } -# The CLI's default scorer, llm_judge, exists only with --judge-model, so name -# static_json as the fallback. The scenario's own scoring_method still wins. +# static_json is also the CLI's default; naming it keeps the fallback explicit. +# The scenario's own scoring_method still wins. judge_args=(--scorer-default static_json) if [ -n "${AOB_JUDGE_MODEL:-}" ]; then judge_args+=(--judge-model "${AOB_JUDGE_MODEL}") diff --git a/docs/evaluation.md b/docs/evaluation.md index 701747910..e85f131bc 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -191,7 +191,7 @@ uv run evaluate \ --trajectories DIR_OR_FILE # required --scenarios FILE [FILE ...] # required, one or more [--reports-dir DIR] # default: reports/ - [--scorer-default NAME] # default: llm_judge + [--scorer-default NAME] # default: static_json [--judge-model MODEL_ID] # required when llm_judge runs [-v] ``` @@ -203,11 +203,13 @@ uv run evaluate \ | ------------- | ---------------------- | ------------------------------------------- | | LLM-As-Judge | `llm_judge` | Works. Installed by passing `--judge-model` | | Code-Based | `static_json` | Works. Deterministic scorer for structured answers | +| Code-Based | `fmea` | Works. Semantic F1 for FMEA catalogue answers ([FMEA Evaluation](fmea-evaluation.md)) | +| Code-Based | `fcc` | Works. Numeric tolerance or set F1 for FCC answers, from `scenario_meta.json` | | Code-Based | `exact_string_match` | **Skeleton — `NotImplementedError`** | | Code-Based | `numeric_match` | **Skeleton — `NotImplementedError`** | | Semantic-Score| `semantic_similarity` | **Skeleton — `NotImplementedError`** | -`static_json` is registered automatically and can be selected with `--scorer-default static_json`. It compares the final answer from a saved trajectory against `scenario.expected_answer`, parsing JSON objects, arrays, Python-style literals, tuple lists, nested structures, markdown-fenced answers, answer-prefixed outputs, and count-only answers. See [Static JSON Evaluation](static-json-evaluation.md) for details. +`static_json` is registered automatically and scores every scenario that names no `scoring_method`, unless `--scorer-default` picks another scorer. It compares the final answer from a saved trajectory against `scenario.expected_answer`, parsing JSON objects, arrays, Python-style literals, tuple lists, nested structures, markdown-fenced answers, answer-prefixed outputs, and count-only answers. See [Static JSON Evaluation](static-json-evaluation.md) for details. Skeleton scorers don't auto-register; calling them raises `NotImplementedError`. Fill in the body and call diff --git a/docs/fmea-evaluation.md b/docs/fmea-evaluation.md index 119954a23..b04c0af36 100644 --- a/docs/fmea-evaluation.md +++ b/docs/fmea-evaluation.md @@ -1,9 +1,12 @@ # FMEA evaluation The `fmea` scorer evaluates structured FMEA catalogue scenarios without an -LLM call. Its primary score is semantic F1, and a scenario passes when its F1 -is strictly greater than `0.70`. The rubric-weighted composite and strict exact -success remain available as secondary diagnostics. +LLM call. Its primary score is semantic F1, and a scenario passes when its F1, +rounded to the 6 decimals it is reported at, is at least the scenario's +`f1_pass_threshold` from `scenario_meta.json` (default `0.80`). A threshold +that is not a number in [0, 1] fails the scenario with a rationale. The +rubric-weighted composite and strict exact success remain available as +secondary diagnostics. For `asset_modes` scenarios, the scorer measures failure-mode set recall, failure-mode set precision, output-contract compliance, and exact diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index 4c9c51818..9386d0abe 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -238,7 +238,19 @@ object. | `extra_keys` | Keys present in model output but not in gold | | `details` | Per-key comparison records | -The scorer marks a scenario as passed only when the structured answer is a strict exact match, and reports `f1` as `score`. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. CAR scenarios with `groundtruth_eval.json` are the exception: see below. +The scorer marks a scenario as passed only when the structured answer is a strict exact match, and reports `f1` as `score`. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. Two kinds of scenario are the exception: those that opt in to a numeric tolerance, and CAR scenarios with `groundtruth_eval.json`. See below. + +### Numeric tolerance + +A scenario can let numbers differ from the gold answer through `scenario_meta.json`: + +```json +{"scoring_method": "static_json", "match": "numeric", "relative_tolerance": 0.1} +``` + +Every flattened gold value must be present. A numeric value is accepted when `|answer - gold| <= relative_tolerance x |gold|`. Any other value, such as an asset id, must match exactly. Keys stay positional, so a list must keep the gold order, and an extra key fails the answer. Comparisons are inclusive and made on values rounded to 6 decimals. The score is F1 over accepted keys, and the per-value results are reported under `tolerance_values`. + +A `match` other than `numeric`, or a missing, negative or non-numeric `relative_tolerance`, fails the scenario with a rationale. Scenarios without a `match` key are scored by strict match as above. ### CAR metrics and pass criteria diff --git a/src/evaluation/runner.py b/src/evaluation/runner.py index 0e36abea2..499b69720 100644 --- a/src/evaluation/runner.py +++ b/src/evaluation/runner.py @@ -13,7 +13,7 @@ def evaluate( *, trajectories_path: Path, scenarios_paths: list[Path], - default_scoring_method: str = "llm_judge", + default_scoring_method: str = "static_json", judge_model: str | None = None, scenario_ids: Collection[str] | None = None, ) -> EvalReport: diff --git a/src/evaluation/scorers/fmea.py b/src/evaluation/scorers/fmea.py index 44d11c46e..edc5ebb84 100644 --- a/src/evaluation/scorers/fmea.py +++ b/src/evaluation/scorers/fmea.py @@ -40,6 +40,17 @@ def _metadata(scenario: Scenario, key: str) -> dict[str, Any]: return value if isinstance(value, dict) else {} +def _f1_pass_threshold(meta: dict[str, Any]) -> float | None: + """Read f1_pass_threshold, defaulting when unset; None when it is not in [0, 1].""" + value = meta.get("f1_pass_threshold") + if value is None: + return _DEFAULT_F1_PASS_THRESHOLD + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + # NaN fails the range test too. + return float(value) if 0.0 <= value <= 1.0 else None + + def _weights(scenario: Scenario) -> dict[str, float]: defaults = { "coverage": 0.35, @@ -183,6 +194,18 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR model = tolerant_model if isinstance(tolerant_model, dict) else {} meta = _metadata(scenario, "scenario_meta") + f1_pass_threshold = _f1_pass_threshold(meta) + if f1_pass_threshold is None: + return ScorerResult( + scorer="fmea", + passed=False, + score=0.0, + rationale=( + "scenario_meta.json f1_pass_threshold must be a number in [0, 1]; " + f"got {meta.get('f1_pass_threshold')!r}." + ), + details={"error": "invalid_scenario_meta"}, + ) tag = str(meta.get("tag", "")) is_asset_modes = tag == "asset_modes" or ( set(gold) == {"asset", "failure_modes"} @@ -201,12 +224,6 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR rubric_weighted_score = sum(weights[name] * dimensions[name] for name in weights) precision, recall = dimensions["precision"], dimensions["coverage"] semantic_f1 = 2.0 * precision * recall / (precision + recall) if precision + recall else 0.0 - threshold_value = meta.get("f1_pass_threshold", _DEFAULT_F1_PASS_THRESHOLD) - f1_pass_threshold = ( - float(threshold_value) - if isinstance(threshold_value, (int, float)) - else _DEFAULT_F1_PASS_THRESHOLD - ) # Compare at the reported precision: 2PR/(P+R) lands just under an exact # boundary (6/7 and 6/8 give 0.7999999999999999), failing a 0.8 threshold. score = round(semantic_f1, 6) diff --git a/src/evaluation/tests/test_fmea_scorer.py b/src/evaluation/tests/test_fmea_scorer.py index 26a1975c1..5ed000812 100644 --- a/src/evaluation/tests/test_fmea_scorer.py +++ b/src/evaluation/tests/test_fmea_scorer.py @@ -149,3 +149,36 @@ def test_f1_above_point_eight_passes_without_requiring_exact_match(): assert result.score == pytest.approx(6 / 7, abs=1e-6) assert result.passed is True assert result.details["strict_success"] is False + + +def _six_of_eight(threshold): + """An answer with F1 6/7 (0.857) against a scenario carrying ``threshold``.""" + scenario = _scenario( + {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(8)}} + ) + if threshold != "unset": + scenario.evaluation_metadata["scenario_meta"]["f1_pass_threshold"] = threshold + answer = {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(6)}} + return fmea_score(scenario, json.dumps(answer), "") + + +def test_scenario_threshold_overrides_the_default(): + result = _six_of_eight(0.9) + assert result.details["f1_pass_threshold"] == 0.9 + assert result.passed is False + + +@pytest.mark.parametrize("threshold", ["unset", None]) +def test_unset_threshold_uses_the_default(threshold): + result = _six_of_eight(threshold) + assert result.details["f1_pass_threshold"] == 0.8 + assert result.passed is True + + +@pytest.mark.parametrize("threshold", [True, -0.1, 1.5, 80, "0.8", float("nan")]) +def test_invalid_threshold_fails_the_scenario_with_a_rationale(threshold): + result = _six_of_eight(threshold) + assert result.passed is False + assert result.score == 0.0 + assert result.details["error"] == "invalid_scenario_meta" + assert "f1_pass_threshold" in result.rationale