fix(evaluation): validate FMEA threshold, fix scorer-default drift

benchmarks/harbor/QUICKSTART.md

@@ -118,8 +118,10 @@ base. `AOB_RUNTIME_IMAGE` is unset in this shell and there is no local
 `assetopsbench/runtime:dev`, or it names an image that is not local. Redo
 step 2 in this shell.
 
-**`unknown scorer 'llm_judge'`**: you are calling `evaluate` by hand without
-`--scorer-default`. Copy what the task's `tests/test.sh` does.
+**`unknown scorer 'llm_judge'`**: a scenario's `scenario_meta.json` routes it
+to `llm_judge`, or `--scorer-default llm_judge` was passed, and no judge model
+was given. Set `AOB_JUDGE_MODEL`, or pass `--judge-model` when calling
+`evaluate` by hand.
 
 **The verifier scores 0 but the agent clearly answered**: check
 `verifier/test-stderr.txt`. The evaluator joins records to scenarios on

benchmarks/harbor/README.md

@@ -307,13 +307,13 @@ on `main`, and a value above what Docker has fails the run
 ## Scoring
 
 `evaluation.loader` takes each scenario's `scoring_method` from
-`scenario_meta.json`, defaulting to `static_json`, and it wins over
-`--scorer-default`. The open profile and mini (whose FMEA scenarios use `fmea`)
-need no judge model. `tests/test.sh` passes `--judge-model` only when
-`AOB_JUDGE_MODEL` is set, and always passes `--scorer-default static_json`,
-because the CLI's own default, `llm_judge`, is registered only with a judge
-model and would otherwise fail every scenario with
-`unknown scorer 'llm_judge'`.
+`scenario_meta.json`, and it wins over `--scorer-default`. A scenario without
+one falls back to `--scorer-default`, which is `static_json`. The open profile
+and mini (whose FMEA scenarios use `fmea`) need no judge model.
+`tests/test.sh` passes `--judge-model` only when `AOB_JUDGE_MODEL` is set, and
+passes `--scorer-default static_json` to keep the fallback explicit. A scenario
+whose `scenario_meta.json` asks for `llm_judge` stops the evaluation with
+`unknown scorer 'llm_judge'` unless a judge model is given.
 
 The generator copies every file the loader reads into
 `tests/scenarios/scenario_N/`: `question.txt`, `groundtruth.txt`,

benchmarks/harbor/template/tests/test.sh

@@ -18,8 +18,8 @@ cd /opt/aob || {
     exit 0
 }
 
-# The CLI's default scorer, llm_judge, exists only with --judge-model, so name
-# static_json as the fallback. The scenario's own scoring_method still wins.
+# static_json is also the CLI's default; naming it keeps the fallback explicit.
+# The scenario's own scoring_method still wins.
 judge_args=(--scorer-default static_json)
 if [ -n "${AOB_JUDGE_MODEL:-}" ]; then
     judge_args+=(--judge-model "${AOB_JUDGE_MODEL}")

docs/evaluation.md

@@ -191,7 +191,7 @@ uv run evaluate \
   --trajectories DIR_OR_FILE     # required
   --scenarios FILE [FILE ...]    # required, one or more
   [--reports-dir DIR]            # default: reports/
-  [--scorer-default NAME]        # default: llm_judge
+  [--scorer-default NAME]        # default: static_json
   [--judge-model MODEL_ID]       # required when llm_judge runs
   [-v]
 ```
@@ -203,11 +203,13 @@ uv run evaluate \
 | ------------- | ---------------------- | ------------------------------------------- |
 | LLM-As-Judge  | `llm_judge`            | Works. Installed by passing `--judge-model` |
 | Code-Based    | `static_json`          | Works. Deterministic scorer for structured answers |
+| Code-Based    | `fmea`                 | Works. Semantic F1 for FMEA catalogue answers ([FMEA Evaluation](fmea-evaluation.md)) |
+| Code-Based    | `fcc`                  | Works. Numeric tolerance or set F1 for FCC answers, from `scenario_meta.json` |
 | Code-Based    | `exact_string_match`   | **Skeleton — `NotImplementedError`**        |
 | Code-Based    | `numeric_match`        | **Skeleton — `NotImplementedError`**        |
 | Semantic-Score| `semantic_similarity`  | **Skeleton — `NotImplementedError`**        |
 
-`static_json` is registered automatically and can be selected with `--scorer-default static_json`. It compares the final answer from a saved trajectory against `scenario.expected_answer`, parsing JSON objects, arrays, Python-style literals, tuple lists, nested structures, markdown-fenced answers, answer-prefixed outputs, and count-only answers. See [Static JSON Evaluation](static-json-evaluation.md) for details.
+`static_json` is registered automatically and scores every scenario that names no `scoring_method`, unless `--scorer-default` picks another scorer. It compares the final answer from a saved trajectory against `scenario.expected_answer`, parsing JSON objects, arrays, Python-style literals, tuple lists, nested structures, markdown-fenced answers, answer-prefixed outputs, and count-only answers. See [Static JSON Evaluation](static-json-evaluation.md) for details.
 
 Skeleton scorers don't auto-register; calling them raises
 `NotImplementedError`. Fill in the body and call

docs/fmea-evaluation.md

@@ -1,9 +1,12 @@
 # FMEA evaluation
 
 The `fmea` scorer evaluates structured FMEA catalogue scenarios without an
-LLM call. Its primary score is semantic F1, and a scenario passes when its F1
-is strictly greater than `0.70`. The rubric-weighted composite and strict exact
-success remain available as secondary diagnostics.
+LLM call. Its primary score is semantic F1, and a scenario passes when its F1,
+rounded to the 6 decimals it is reported at, is at least the scenario's
+`f1_pass_threshold` from `scenario_meta.json` (default `0.80`). A threshold
+that is not a number in [0, 1] fails the scenario with a rationale. The
+rubric-weighted composite and strict exact success remain available as
+secondary diagnostics.
 
 For `asset_modes` scenarios, the scorer measures failure-mode set recall,
 failure-mode set precision, output-contract compliance, and exact

docs/static-json-evaluation.md

@@ -238,7 +238,19 @@ object.
 | `extra_keys`                   | Keys present in model output but not in gold                                          |
 | `details`                      | Per-key comparison records                                                            |
 
-The scorer marks a scenario as passed only when the structured answer is a strict exact match, and reports `f1` as `score`. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. CAR scenarios with `groundtruth_eval.json` are the exception: see below.
+The scorer marks a scenario as passed only when the structured answer is a strict exact match, and reports `f1` as `score`. Partial correctness is still available through `score`, `f1`, `partial_exact_match_accuracy`, and detailed key-level results. Two kinds of scenario are the exception: those that opt in to a numeric tolerance, and CAR scenarios with `groundtruth_eval.json`. See below.
+
+### Numeric tolerance
+
+A scenario can let numbers differ from the gold answer through `scenario_meta.json`:
+
+```json
+{"scoring_method": "static_json", "match": "numeric", "relative_tolerance": 0.1}
+```
+
+Every flattened gold value must be present. A numeric value is accepted when `|answer - gold| <= relative_tolerance x |gold|`. Any other value, such as an asset id, must match exactly. Keys stay positional, so a list must keep the gold order, and an extra key fails the answer. Comparisons are inclusive and made on values rounded to 6 decimals. The score is F1 over accepted keys, and the per-value results are reported under `tolerance_values`.
+
+A `match` other than `numeric`, or a missing, negative or non-numeric `relative_tolerance`, fails the scenario with a rationale. Scenarios without a `match` key are scored by strict match as above.
 
 ### CAR metrics and pass criteria
 

src/evaluation/runner.py

@@ -13,7 +13,7 @@ def evaluate(
     *,
     trajectories_path: Path,
     scenarios_paths: list[Path],
-    default_scoring_method: str = "llm_judge",
+    default_scoring_method: str = "static_json",
     judge_model: str | None = None,
     scenario_ids: Collection[str] | None = None,
 ) -> EvalReport:

src/evaluation/scorers/fmea.py

@@ -40,6 +40,17 @@ def _metadata(scenario: Scenario, key: str) -> dict[str, Any]:
     return value if isinstance(value, dict) else {}
 
 
+def _f1_pass_threshold(meta: dict[str, Any]) -> float | None:
+    """Read f1_pass_threshold, defaulting when unset; None when it is not in [0, 1]."""
+    value = meta.get("f1_pass_threshold")
+    if value is None:
+        return _DEFAULT_F1_PASS_THRESHOLD
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    # NaN fails the range test too.
+    return float(value) if 0.0 <= value <= 1.0 else None
+
+
 def _weights(scenario: Scenario) -> dict[str, float]:
     defaults = {
         "coverage": 0.35,
@@ -183,6 +194,18 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR
 
     model = tolerant_model if isinstance(tolerant_model, dict) else {}
     meta = _metadata(scenario, "scenario_meta")
+    f1_pass_threshold = _f1_pass_threshold(meta)
+    if f1_pass_threshold is None:
+        return ScorerResult(
+            scorer="fmea",
+            passed=False,
+            score=0.0,
+            rationale=(
+                "scenario_meta.json f1_pass_threshold must be a number in [0, 1]; "
+                f"got {meta.get('f1_pass_threshold')!r}."
+            ),
+            details={"error": "invalid_scenario_meta"},
+        )
     tag = str(meta.get("tag", ""))
     is_asset_modes = tag == "asset_modes" or (
         set(gold) == {"asset", "failure_modes"}
@@ -201,12 +224,6 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR
     rubric_weighted_score = sum(weights[name] * dimensions[name] for name in weights)
     precision, recall = dimensions["precision"], dimensions["coverage"]
     semantic_f1 = 2.0 * precision * recall / (precision + recall) if precision + recall else 0.0
-    threshold_value = meta.get("f1_pass_threshold", _DEFAULT_F1_PASS_THRESHOLD)
-    f1_pass_threshold = (
-        float(threshold_value)
-        if isinstance(threshold_value, (int, float))
-        else _DEFAULT_F1_PASS_THRESHOLD
-    )
     # Compare at the reported precision: 2PR/(P+R) lands just under an exact
     # boundary (6/7 and 6/8 give 0.7999999999999999), failing a 0.8 threshold.
     score = round(semantic_f1, 6)

src/evaluation/tests/test_fmea_scorer.py

@@ -149,3 +149,36 @@ def test_f1_above_point_eight_passes_without_requiring_exact_match():
     assert result.score == pytest.approx(6 / 7, abs=1e-6)
     assert result.passed is True
     assert result.details["strict_success"] is False
+
+
+def _six_of_eight(threshold):
+    """An answer with F1 6/7 (0.857) against a scenario carrying ``threshold``."""
+    scenario = _scenario(
+        {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(8)}}
+    )
+    if threshold != "unset":
+        scenario.evaluation_metadata["scenario_meta"]["f1_pass_threshold"] = threshold
+    answer = {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(6)}}
+    return fmea_score(scenario, json.dumps(answer), "")
+
+
+def test_scenario_threshold_overrides_the_default():
+    result = _six_of_eight(0.9)
+    assert result.details["f1_pass_threshold"] == 0.9
+    assert result.passed is False
+
+
+@pytest.mark.parametrize("threshold", ["unset", None])
+def test_unset_threshold_uses_the_default(threshold):
+    result = _six_of_eight(threshold)
+    assert result.details["f1_pass_threshold"] == 0.8
+    assert result.passed is True
+
+
+@pytest.mark.parametrize("threshold", [True, -0.1, 1.5, 80, "0.8", float("nan")])
+def test_invalid_threshold_fails_the_scenario_with_a_rationale(threshold):
+    result = _six_of_eight(threshold)
+    assert result.passed is False
+    assert result.score == 0.0
+    assert result.details["error"] == "invalid_scenario_meta"
+    assert "f1_pass_threshold" in result.rationale