diff --git a/policybench/audit.py b/policybench/audit.py index a2699761..00105fb7 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -49,6 +49,12 @@ # is a parse failure, not a substantive error. MISSING_VALUE = "missing" +# Which wrong models a judged case needs with no value before its case note may +# say parse_contract_failure (decision d1514): "any" keeps the judge's label +# while at least one of them returned no value; "all" keeps it only when every +# one did, which no judged case has (collect_audit classifies those itself). +CASE_PARSE_FAILURE_NEEDS_MISSING = "any" + @dataclass(frozen=True) class WrongModel: @@ -658,6 +664,23 @@ def _row_failure_source(meta: dict, model: str, requested_source: str) -> str: return source +def _case_failure_source(meta: dict, case_source: str) -> str: + """Keep a judged case's parse-contract label to cases with missing values. + + ``case_source`` is the judge's case class after collect_audit's + budget-exhaustion check. Like a row, a case note says parse_contract_failure + only when its wrong models returned no value: at least one of them, or all + of them, as :data:`CASE_PARSE_FAILURE_NEEDS_MISSING` says. + """ + if case_source != "parse_contract_failure": + return case_source + missing_models = set(meta.get("missing_models", [])) + needs = {"any": any, "all": all}[CASE_PARSE_FAILURE_NEEDS_MISSING] + if needs(model in missing_models for model in meta.get("wrong_models", [])): + return case_source + return "llm_error" + + def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame]: """Fold verdicts into the annotation schema. @@ -782,7 +805,9 @@ def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame] "scenario_id": meta["scenario_id"], "variable": meta["variable"], "wrong_model_count": len(meta["wrong_models"]), - "case_failure_source": case_source, + # Only the case note: a row the verdict leaves unlabelled + # still falls back to the judge's case class above. + "case_failure_source": _case_failure_source(meta, case_source), "case_failure_subtype": case_subtype, "reference_suspect": reference_suspect, "reference_bug_hypothesis": str( diff --git a/tests/test_audit.py b/tests/test_audit.py index a52b2a67..b21cd718 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -4,16 +4,23 @@ import json from pathlib import Path +from unittest import mock import pandas as pd import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +from policybench import audit from policybench.annotation_taxonomy import ( FAILURE_SOURCE_VALUES, FAILURE_SUBTYPE_VALUES, ) from policybench.audit import ( AUDIT_OUTPUT_SCHEMA, + MISSING_VALUE, + AuditCase, + WrongModel, _row_failure_source, build_audit_cases, collect_audit, @@ -23,6 +30,7 @@ render_case_prompt, ) from policybench.judge_template import CURRENT_TEMPLATE_VERSION +from policybench.spec import metric_type_for_output @pytest.fixture @@ -593,6 +601,244 @@ def test_collect_audit_coerces_parse_source_for_parsed_predictions(tmp_path: Pat assert out["row"].iloc[0]["failure_source"] == "llm_error" +def _parsed(model: str) -> WrongModel: + return WrongModel(model, "$250.00", "Derived 0, submitted 250. value = 250") + + +def _missing(model: str, recorded: str = "") -> WrongModel: + return WrongModel(model, MISSING_VALUE, "", failure_source=recorded) + + +def _judged_case( + audit_dir: Path, + wrong_models: list[WrongModel], + case_source: str, + model_sources: dict[str, str], + case_subtype: str = "other", + model_subtypes: dict[str, str] | None = None, +) -> dict: + """Write one case's manifest row and verdict as prepare_audit and a judge + leave them, and return the manifest row. A model absent from + ``model_sources`` has no per-model entry in the verdict.""" + case = AuditCase( + case_id="us__s0__snap", + country="us", + scenario_id="s0", + variable="snap", + metric_type=metric_type_for_output("snap"), + reference_value="$0.00", + reference_derivation="", + question="", + wrong_models=tuple(wrong_models), + ) + meta = case.to_manifest_row() + case_dir = audit_dir / "cases" / case.case_id + case_dir.mkdir(parents=True, exist_ok=True) + (audit_dir / "cases.jsonl").write_text(json.dumps(meta) + "\n") + verdict = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": case_source, + "case_failure_subtype": case_subtype, + "rationale": "The submitted value contradicts the derivation.", + "models": [ + { + "model": model, + "failure_source": source, + "failure_subtype": (model_subtypes or {}).get(model, "other"), + "diagnosis": f"{model} submitted a value its derivation rejects.", + } + for model, source in model_sources.items() + ], + } + (case_dir / "verdict.json").write_text(json.dumps(verdict)) + return meta + + +def _collect(audit_dir: Path) -> dict[str, pd.DataFrame]: + return collect_audit(audit_dir.parent / "us", audit_dir) + + +def test_judged_case_label_needs_a_missing_model(tmp_path: Path): + """The published shape of 012, 015 and 044 at bc305793: the only wrong model + derived 0, submitted 1, and the judge called case and row a parse failure.""" + audit_dir = tmp_path / "audit" + meta = _judged_case( + audit_dir, + [_parsed("m1")], + "parse_contract_failure", + {"m1": "parse_contract_failure"}, + ) + assert not meta["parse_failure_only"] + + out = _collect(audit_dir) + + assert out["row"]["failure_source"].tolist() == ["llm_error"] + note = out["case"].iloc[0] + assert note["case_failure_source"] == "llm_error" + assert note["case_failure_subtype"] == "other" + assert note["case_annotation"] == "The submitted value contradicts the derivation." + + +@pytest.mark.parametrize( + ("needs", "case_source"), + [("any", "parse_contract_failure"), ("all", "llm_error")], +) +def test_mixed_case_label_follows_the_d1514_option( + tmp_path: Path, needs: str, case_source: str +): + """080 head CHIP at bc305793: one model returned nothing, one derived 0 and + submitted 1. Option 1 ("any") keeps the judge's parse failure; option 2 + ("all") makes it an LLM error. The rows are the same either way.""" + audit_dir = tmp_path / "audit" + _judged_case( + audit_dir, + [_parsed("claude"), _missing("glm")], + "parse_contract_failure", + {"claude": "llm_error", "glm": "parse_contract_failure"}, + model_subtypes={"glm": "missing_output"}, + ) + + with mock.patch.object(audit, "CASE_PARSE_FAILURE_NEEDS_MISSING", needs): + out = _collect(audit_dir) + + assert out["case"].iloc[0]["case_failure_source"] == case_source + rows = out["row"].set_index("model") + assert rows["failure_source"].to_dict() == { + "claude": "llm_error", + "glm": "parse_contract_failure", + } + assert rows.loc["glm", "failure_subtype"] == "missing_output" + + +def test_case_label_may_match_no_row_while_a_model_is_missing(tmp_path: Path): + """Intended under option 1, and why no property says a case's class is + among its rows' (sol_review_r3 finding 4): a parsed miss becomes an LLM + error and a missing answer keeps its recorded ceiling exhaustion, so the + judge's case-level parse failure stands while no row carries it.""" + audit_dir = tmp_path / "audit" + _judged_case( + audit_dir, + [_parsed("m1"), _missing("m2", recorded="budget_exhausted_at_ceiling")], + "parse_contract_failure", + {"m1": "parse_contract_failure", "m2": "parse_contract_failure"}, + ) + + out = _collect(audit_dir) + + assert out["row"].set_index("model")["failure_source"].to_dict() == { + "m1": "llm_error", + "m2": "budget_exhausted_at_ceiling", + } + assert out["case"].iloc[0]["case_failure_source"] == "parse_contract_failure" + + +def test_unlabelled_row_falls_back_to_the_judges_case_class(tmp_path: Path): + """The demotion is the case note's only: a model the verdict leaves + unlabelled takes the judge's case class, so a missing model stays a parse + failure even when option 2 makes its case an LLM error.""" + audit_dir = tmp_path / "audit" + _judged_case( + audit_dir, + [_parsed("m1"), _missing("m2")], + "parse_contract_failure", + {}, + ) + + with mock.patch.object(audit, "CASE_PARSE_FAILURE_NEEDS_MISSING", "all"): + out = _collect(audit_dir) + + assert out["case"].iloc[0]["case_failure_source"] == "llm_error" + assert out["row"].set_index("model")["failure_source"].to_dict() == { + "m1": "llm_error", + "m2": "parse_contract_failure", + } + + +def test_other_judged_case_classes_publish_unchanged(tmp_path: Path): + for source in FAILURE_SOURCE_VALUES: + if source in ("parse_contract_failure", "budget_exhausted_at_ceiling"): + continue + audit_dir = tmp_path / source / "audit" + _judged_case(audit_dir, [_parsed("m1")], source, {"m1": source}) + assert _collect(audit_dir)["case"].iloc[0]["case_failure_source"] == source + + +_MODELS = ["m1", "m2", "m3", "m4"] + + +@st.composite +def _judged_cases(draw): + """A case of one to four wrong models, each with or without a value (a + missing one may record ceiling exhaustion), and any verdict the schema + allows: any case class and, per model, any class and subtype or no + entry at all.""" + wrong = [] + for model in _MODELS[: draw(st.integers(1, len(_MODELS)))]: + if draw(st.booleans()): + wrong.append(_parsed(model)) + else: + recorded = draw(st.sampled_from(["", "budget_exhausted_at_ceiling"])) + wrong.append(_missing(model, recorded)) + sources = st.sampled_from(FAILURE_SOURCE_VALUES) + subtypes = st.sampled_from(FAILURE_SUBTYPE_VALUES) + labelled = [m.model for m in wrong if draw(st.booleans())] + return { + "wrong_models": wrong, + "case_source": draw(sources), + "case_subtype": draw(subtypes), + "model_sources": {model: draw(sources) for model in labelled}, + "model_subtypes": {model: draw(subtypes) for model in labelled}, + } + + +@settings(max_examples=300, deadline=None) +@given(case=_judged_cases()) +def test_judged_case_parse_label_properties(tmp_path_factory, case): + """For any judged case: + - option 1 (the default): with no wrong model in missing_models, the + published case_failure_source is never parse_contract_failure; + - option 2: with any wrong model holding a value, it never is; + - each option changes only case_failure_source, and only from the + judge's parse_contract_failure to llm_error: the rows, and every other + case column, are what collect_audit published before the rule. + """ + audit_dir = tmp_path_factory.mktemp("case") / "audit" + meta = _judged_case(audit_dir, **case) + missing = set(meta["missing_models"]) + wrong = meta["wrong_models"] + + with mock.patch.object(audit, "_case_failure_source", lambda meta, s: s): + before = _collect(audit_dir) + after = {} + for needs in ("any", "all"): + with mock.patch.object(audit, "CASE_PARSE_FAILURE_NEEDS_MISSING", needs): + after[needs] = _collect(audit_dir) + pd.testing.assert_frame_equal(_collect(audit_dir)["case"], after["any"]["case"]) + + source = after["any"]["case"].iloc[0]["case_failure_source"] + if meta["parse_failure_only"]: + # Classified without a judge; the rule does not reach these. + assert source == before["case"].iloc[0]["case_failure_source"] + elif not missing: + assert source != "parse_contract_failure" + if not meta["parse_failure_only"]: + assert after["all"]["case"].iloc[0]["case_failure_source"] != ( + "parse_contract_failure" + ) + + judged = before["case"].iloc[0]["case_failure_source"] + for needs, out in after.items(): + pd.testing.assert_frame_equal(out["row"], before["row"]) + others = [c for c in before["case"].columns if c != "case_failure_source"] + pd.testing.assert_frame_equal(out["case"][others], before["case"][others]) + published = out["case"].iloc[0]["case_failure_source"] + if published != judged: + assert (judged, published) == ("parse_contract_failure", "llm_error") + kept = (any if needs == "any" else all)(m in missing for m in wrong) + assert not kept + + def test_parse_verdict_survives_brace_in_leading_prose(tmp_path: Path): f = tmp_path / "verdict.json" f.write_text(