diff --git a/docs/audit.md b/docs/audit.md index 3c3ea533..8f8998c5 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -18,7 +18,8 @@ Python (`policybench/audit.py`). # 1. Assemble one prompt per wrong case (+ shared output schema + manifest). uv run python -m policybench.cli audit-prepare \ --country-dir results//us \ - --audit-dir results//us/audit + --audit-dir results//us/audit \ + --template-version 2 # 2. Classify in bulk via Codex (ChatGPT plan). Resumable + parallel. AUDIT_PARALLEL=4 AUDIT_REASONING_EFFORT=low \ @@ -47,6 +48,122 @@ side. It is told to default to `llm_error` and flag `reference_suspect` only with concrete evidence — the same conservatism as the deterministic inferrer, but with actual tax/benefit reasoning instead of keyword matching. +## Judge template versions + +Each prompt starts with a fixed header, the judge template, followed by the +case (`policybench/judge_template.py`). The template is versioned because a +release carries a seed verdict forward only when its case re-renders to the +exact bytes the judge read. + +| Version | Judged | Change | +|---|---|---| +| v1 | every verdict through release dashboard-data-20261010 (Claude Haiku 5.5), the last release judged before versions existed | none | +| v2 | the version a new audit names | drops v1's claim that the reference pipeline "has survived an adversarial review program" and that "the few real bugs found were fixed before this run" | + +The v1 claim is not so: release dashboard-data-20261006's exclusion record +stops scoring 28 outputs for engine defects, and the 2026-10-05 reference +adversary confirmed four more (`reference_audit/2026-10-05-reference-adversary/`, +#200). + +- Version selection is explicit. `render_case_prompt` and `prepare_audit` take + `template_version` as a required keyword, and `audit-prepare` requires + `--template-version`; nothing defaults to `CURRENT_TEMPLATE_VERSION`. A + judged case renders on the version its sidecar records. Nothing reads a + version off a prompt to choose how to render it: `template_version_of` only + records what a judge read and checks a tree. +- Both runners have the judge read a private copy of `prompt.md`, and record + that copy's sha256 (`prompt_sha256`) and version (`judge_template_version`) + in `verdict.meta.json`. A verdict is not published if `prompt.md` no longer + holds those bytes. A prompt that begins with no version's header, one + `audit-prepare` did not write, records `null`. Before it judges a case, a + runner removes the case's invalid `verdict.json` and stale sidecar, and it + publishes the sidecar before the verdict. An interrupted publish can leave + a sidecar without a verdict, which the next run removes, but never a new + verdict without its sidecar. +- A sidecar without the field was judged on v1 + (`UNRECORDED_TEMPLATE_VERSION`). That is a fixed rule for sidecars written + before versions existed, not a guess from the prompt. +- A verdict without a sidecar has no record of what its judge read. + `audit-prepare` re-opens it and `audit-collect` refuses it. A runner's + verdict can lose its sidecar to an `audit-prepare` racing it, so this is + what keeps such a verdict out. Every verdict in releases 20260929 and + 20260930 has a sidecar. +- `audit-prepare` renders a case that already has a verdict with the version + its sidecar records. The verdict stands only if that reproduces + `prompt.md` byte for byte and, when the sidecar records `prompt_sha256` + (the hash of the bytes its judge read), matches it. Without `prompt.md`, + the verdict stands only if that hash matches, and the prompt is restored. + A sidecar that records `verdict_sha256` must also describe the verdict. A + kept case's files are not rewritten. Otherwise the case changed since it + was judged, or its version or judged bytes are unknown, and the case is + re-opened: the verdict is dropped, and so is its sidecar when the sidecar + describes that verdict or records no `verdict_sha256`. A sidecar that + describes another verdict when `audit-prepare` reads it, such as the + record of a verdict a runner is still publishing, is left in place. A + `prompt.md` on another version is re-opened, not adopted. New and + re-opened cases use the version the caller names. +- `audit-prepare` may run while a runner judges the same tree, with one + runner per case and one `audit-prepare` at a time. Once both finish, no + verdict passes `audit-collect`, or survives the next `audit-prepare`, on + bytes its judge did not read. Two rules carry that guarantee. First, a + verdict without its sidecar is refused and re-opened. Second, a sidecar + that survives is the runner's own record of the bytes its judge read, so + a verdict whose case `audit-prepare` rewrote meanwhile fails the hash + check: `audit-collect` refuses it and `audit-prepare` re-opens it. The + other measures make a stranded verdict rare without ruling it out. + Runners start each case clean and publish the sidecar first, and + `audit-prepare` removes a sidecar only when it describes the verdict being + dropped. But `audit-prepare` reads a sidecar before it removes it, so a + runner publishing in between can still lose its sidecar; its verdict is + then refused. Run `audit-collect` after both finish. +- Each release driver names the version its new and re-opened cases render + on, as `JUDGE_TEMPLATE_VERSION`. The drivers of releases judged before + versions existed (`scripts/finish_adds0928.py`, `scripts/finish_gpt61sol.py`, + `scripts/finish_haiku55.py`) name v1, so re-running one reproduces its + release's prompts. A new release's driver names v2. Each driver copies the + seed's prompts, verdicts and sidecars into its stage before it calls + `prepare_audit`, so each seed case renders on its recorded version and a + kept case's prompt keeps the seed's bytes. +- A tree may mix versions. `audit-collect` refuses a tree in which a verdict + records an unknown version, or one other than the version its `prompt.md` + begins with (`policybench.audit.template_version_problems`). It also + refuses a verdict whose sidecar's `prompt_sha256` is not `prompt.md`'s + bytes (`prompt_hash_problems`), such as a verdict published after an + `audit-prepare` that raced its runner rewrote the case. Re-running + `audit-prepare` re-opens those cases. +- A version's text never changes once a verdict is judged on it; changing the + template means adding a version. `tests/test_judge_template.py` pins each + version's sha256. +- Every writer of `verdict.meta.json` must record the field. A writer that + does not, such as the Workflow and session judges of + `results/local/adds202609/judge_stages.py`, labels its verdicts v1; on a v2 + prompt, `audit-collect` refuses them and `audit-prepare` re-opens them. + +### Invariants + +`tests/test_judge_template.py` states and tests these: + +- **Every committed prompt re-renders on its recorded version.** Each prompt + sha256 committed in `docs/gpt61sol/` and `docs/haiku55/` (the seed digests + and the judge provenance records) re-renders on the version its verdict + records, v1. It renders from the committed board of the release that judged + it: release 20260929, 20260930 or 20261010. Release 20261006 judged + nothing. It carried release 20260930's audit while rewording nine reference + explanations, so those nine prompts do not re-render from its own board; + release 20261010 re-opened them. This test is local only: it needs the + audit grounding, which is not committed. In CI, property tests check that + `prepare_audit` keeps a judged case's files only on the exact bytes its + judge read. +- **v2 differs from v1 only by the dropped clause.** The header diff is one + deletion, `V1_REVIEW_CLAIM`. For any case, the two prompts differ only in + their headers, and the v2 prompt is the v1 prompt without that clause. +- **Version selection is explicit, never inferred.** The renderers have no + default version, and every call in `policybench/` and `scripts/` names one. + `prepare_audit` decides whether to keep a verdict from the recorded version + alone: a prompt on any other version is re-opened, never adopted. +- No header is a prefix of another, so a prompt's version is unambiguous. +- `prepare_audit` is idempotent, and the tree it writes validates. + ## Acting on suspect references Cases with `reference_suspect=true` are candidate PolicyEngine or data bugs. diff --git a/policybench/audit.py b/policybench/audit.py index 0c947b1d..a2699761 100644 --- a/policybench/audit.py +++ b/policybench/audit.py @@ -19,6 +19,7 @@ from __future__ import annotations +import hashlib import json import re import shutil @@ -35,6 +36,13 @@ ) from policybench.case_annotations import _format_value, wrong_prediction_rows from policybench.full_run_export import load_case_reference_explanations +from policybench.judge_template import ( + JUDGE_TEMPLATE_HEADERS, + TEMPLATE_VERSION_FIELD, + recorded_template_version, + template_header, + template_version_of, +) from policybench.spec import metric_type_for_output # _format_value renders a missing/NaN prediction as this sentinel; such a miss @@ -275,67 +283,21 @@ def _scenario_questions( ], } -_PROMPT_HEADER = """\ -You are diagnosing why AI models missed a PolicyEngine reference value on a \ -US/UK tax-and-benefit estimation benchmark. The models answered from \ -parametric knowledge with no tools; PolicyEngine's microsimulation is the \ -reference. - -The reference value and its derivation are generated directly from the \ -engine's computation trace, and the reference pipeline has survived an \ -adversarial review program: every wrong-reference hypothesis raised by \ -earlier audits was adjudicated against primary sources, and the few real \ -bugs found were fixed before this run. Treat the reference and its \ -derivation as correct. Your job is NOT to re-litigate the reference — it is \ -to explain each model's mistake decisively. - -For every wrong model, write a `diagnosis`: 1-3 definitive sentences naming \ -the exact rule, eligibility pathway, deduction, threshold, or computation \ -step that model missed or misapplied, grounded in its own stated reasoning. \ -Be specific — "treated the 138% FPL MAGI limit as the only Medicaid pathway \ -and never applied the aged/disabled income test, which deducts the Medicare \ -Part B premium from countable income" — not generic ("got the income \ -calculation wrong"). If the model gave no usable reasoning, derive the \ -mistake from its answer: state what the correct derivation yields and what \ -shortcut the model's number is consistent with. - -Hedging is forbidden in `diagnosis` and `rationale`, and these phrasings \ -are mechanically rejected: "plausible"; "not enough evidence"; \ -"insufficient evidence/information"; "cannot/unable to \ -determine/verify/confirm/tell"; "difficult/hard to verify"; "may be/have"; \ -"might be/have"; "possibly"; "perhaps"; "unclear"; "the reference \ -is/appears/seems correct" or any other verdict on the reference. Write \ -definitively around them (e.g. "excess shelter costs are deductible", not \ -"may be deducted"). Doubt \ -about the reference belongs ONLY in reference_suspect + \ -reference_bug_hypothesis, and requires a concrete contradiction: the \ -derivation conflicts with a specific statute, regulation, or published \ -parameter you can name, or contradicts its own arithmetic. Absent that, set \ -reference_suspect=false and diagnose from the reference as ground truth. - -failure_source meanings: -- llm_error: the model reasoned or computed incorrectly (the usual case). -- prompt_ambiguity: the question is genuinely ambiguous; a careful expert \ -could read it more than one way. Name the two readings. -- reference_model_issue_fixed / reference_data_issue_fixed: the reference \ -value looks wrong (PolicyEngine logic / underlying data). Use with \ -reference_suspect=true and a concrete contradiction. -- parse_contract_failure: the model's answer was missing or unparseable, not a \ -substantive error. -- budget_exhausted_at_ceiling: the provider length-terminated every retry through \ -its maximum allowed completion budget. -- needs_review: genuinely cannot tell; name precisely what information is \ -missing. - -Output ONLY the JSON verdict matching the schema. Do not run any commands; all \ -information you need is below. -""" +# The judge's prompt is a versioned header (policybench.judge_template) and +# the case. A verdict is carried only on the exact bytes it was judged on, so +# a seed case renders with the version its sidecar records. Every caller names +# the version: nothing here picks one by default or reads one off a prompt. + +def render_case_prompt(case: AuditCase, *, template_version: int) -> str: + """Render the self-contained classification prompt for one case. -def render_case_prompt(case: AuditCase) -> str: - """Render the self-contained classification prompt for one case.""" + ``template_version`` picks the header (``JUDGE_TEMPLATE_HEADERS``); the + rest of the prompt does not depend on it. It has no default: a release + reproduces its prompts only on the version its verdicts were judged on. + """ lines = [ - _PROMPT_HEADER, + template_header(template_version), f"\nCOUNTRY: {case.country.upper()}", f"OUTPUT (variable): {case.variable} [{case.metric_type}]", f"POLICYENGINE REFERENCE VALUE: {case.reference_value}", @@ -368,6 +330,8 @@ def prepare_audit( country_dir: Path, audit_dir: Path, grounding_lookup: dict[tuple[str, str], str] | None = None, + *, + template_version: int, ) -> list[AuditCase]: """Write per-case prompts, the shared output schema, and a manifest. @@ -376,8 +340,24 @@ def prepare_audit( schema.json cases.jsonl cases//prompt.md - cases//verdict.json (written later by the runner) + cases//verdict.json (written later by the runner) + cases//verdict.meta.json (its provenance sidecar) + + A case that already has a verdict (a seed a release driver copied in, or + an earlier run's) keeps it only on the exact bytes its judge read + (:func:`_judged_prompt`): the case rendered on the template version its + sidecar records (absent: v1) must equal its prompt.md byte for byte, or, + when prompt.md is missing, hash to the ``prompt_sha256`` its sidecar + records. A kept case's files are left untouched. Otherwise the case + changed since it was judged, or its version or judged bytes are unknown: + the verdict and its sidecar are dropped and the case is re-opened. New + and re-opened cases are rendered with ``template_version``, which the + caller must name. The version a case renders with comes from the sidecar + or the caller, never from prompt.md: a prompt that begins with another + version's header is not adopted. Prompts are written as UTF-8 bytes, with + no newline translation. """ + template_header(template_version) cases = build_audit_cases(country_dir, grounding_lookup=grounding_lookup) audit_dir.mkdir(parents=True, exist_ok=True) (audit_dir / "schema.json").write_text(json.dumps(AUDIT_OUTPUT_SCHEMA, indent=2)) @@ -405,24 +385,189 @@ def prepare_audit( case_dir.mkdir(exist_ok=True) prompt_path = case_dir / "prompt.md" verdict_path = case_dir / "verdict.json" - new_prompt = render_case_prompt(case) - # Content-aware resumability: if the case changed since it was last - # classified (e.g. a model was re-run and now answers differently), - # the prompt differs from the stored one — drop the stale verdict so - # the runner re-classifies it rather than reusing the old label. - if ( - verdict_path.exists() - and prompt_path.exists() - and prompt_path.read_text() != new_prompt - ): - verdict_path.unlink() - # The provenance sidecar describes that verdict; a re-judge by - # the other runner must not inherit it. - (case_dir / "verdict.meta.json").unlink(missing_ok=True) - prompt_path.write_text(new_prompt) + verdict = _read_or_none(verdict_path) + if verdict is not None: + judged = _judged_prompt(case, case_dir, verdict) + if judged is not None: + # The verdict stands on these bytes. A prompt.md the + # sidecar's hash vouched for is restored. + if not prompt_path.exists(): + prompt_path.write_bytes(judged) + continue + # Content-aware resumability: the case changed since it was + # classified (e.g. a model was re-run and now answers + # differently), or its template or judged bytes are unknown. + # Drop the stale verdict so the runner re-classifies it rather + # than reusing the old label. + _drop_verdict(case_dir, verdict) + prompt = render_case_prompt(case, template_version=template_version) + prompt_path.write_bytes(prompt.encode("utf-8")) return cases +def _read_or_none(path: Path) -> bytes | None: + """A file's bytes, or None when it does not exist (a runner may remove a + file between a check and a read).""" + try: + return path.read_bytes() + except FileNotFoundError: + return None + + +def _judged_prompt(case: AuditCase, case_dir: Path, verdict: bytes) -> bytes | None: + """The prompt bytes the judged case's ``verdict`` stands on, or None when + the case must be re-opened. + + A sidecar that records ``verdict_sha256`` must describe ``verdict``: + otherwise it is another verdict's record, such as one a runner has + published and whose verdict has not arrived yet. The case is rendered on + the version the sidecar records (absent: v1; an unknown version, or no + sidecar, gives None). Those bytes must be prompt.md's exactly, and, when the sidecar + records ``prompt_sha256`` (the hash of the bytes its judge read), hash to + it. Without prompt.md, that hash is the only evidence of what the judge + read, so a sidecar without one gives None. + """ + meta = _sidecar(case_dir) + recorded_verdict = (meta or {}).get("verdict_sha256") + if recorded_verdict is not None and ( + recorded_verdict != hashlib.sha256(verdict).hexdigest() + ): + return None + judged_on = recorded_template_version(meta) + if judged_on is None: + return None + rendered = render_case_prompt(case, template_version=judged_on).encode("utf-8") + recorded_sha256 = (meta or {}).get("prompt_sha256") + if recorded_sha256 is not None and ( + recorded_sha256 != hashlib.sha256(rendered).hexdigest() + ): + return None + prompt_path = case_dir / "prompt.md" + if prompt_path.exists(): + return rendered if prompt_path.read_bytes() == rendered else None + return rendered if recorded_sha256 is not None else None + + +def _drop_verdict(case_dir: Path, verdict: bytes) -> None: + """Re-open a case: remove ``verdict``, the verdict prepare_audit read, + and its sidecar. + + The sidecar goes only when it describes ``verdict`` (its + ``verdict_sha256``) or records no verdict_sha256, as sidecars from before + runners recorded one do; a re-judge by the other runner must not inherit + it. Any other sidecar is the record of a verdict a runner is publishing, + which runners do sidecar first: removing it would leave that verdict with + no record of the bytes its judge read, and runners remove a sidecar left + without a verdict themselves. Likewise verdict.json goes only while it + still holds ``verdict``: one a runner has published since is left for the + collect gates and the next preparation to judge. + + This narrows the race without closing it: the sidecar is read before it + is removed, so a runner publishing in between can still lose its own. + Such a verdict has no sidecar, so _judged_prompt re-opens it and + template_version_problems refuses it. + """ + verdict_path = case_dir / "verdict.json" + if _read_or_none(verdict_path) == verdict: + verdict_path.unlink(missing_ok=True) + meta = _sidecar(case_dir) + if meta is None: + return + recorded = meta.get("verdict_sha256") + if recorded is None or recorded == hashlib.sha256(verdict).hexdigest(): + (case_dir / "verdict.meta.json").unlink(missing_ok=True) + + +def _sidecar(case_dir: Path) -> dict | None: + """A case's ``verdict.meta.json``, or None when it has none. + + A sidecar that is not a JSON object records no template version. + """ + text = _read_or_none(case_dir / "verdict.meta.json") + if text is None: + return None + try: + meta = json.loads(text) + except ValueError: + meta = None + return meta if isinstance(meta, dict) else {TEMPLATE_VERSION_FIELD: None} + + +def template_version_problems(audit_dir: Path) -> list[tuple[str, str]]: + """Each judged case whose prompt and verdict disagree on the template. + + A tree may mix versions: a release's carried seeds keep the version they + were judged on, while its new and re-opened cases use the one its driver + names. Every verdict must have its sidecar and its prompt.md, its + sidecar must name a known version (absent: v1), and prompt.md must begin + with that version's header. A verdict without a sidecar has no record of + the bytes its judge read: one a runner published can lose its sidecar to + an audit-prepare racing it. Returns ``(case_id, problem)`` for each case + that fails, in case order. + """ + problems: list[tuple[str, str]] = [] + cases_root = audit_dir / "cases" + if not cases_root.is_dir(): + return problems + for case_dir in sorted(cases_root.iterdir()): + if not (case_dir / "verdict.json").is_file(): + continue + meta = _sidecar(case_dir) + recorded = recorded_template_version(meta) + prompt_path = case_dir / "prompt.md" + if meta is None: + problem = "a verdict without its sidecar (verdict.meta.json)" + elif recorded is None: + problem = ( + f"its sidecar's {TEMPLATE_VERSION_FIELD} " + f"{meta.get(TEMPLATE_VERSION_FIELD)!r} names no template " + f"version ({sorted(JUDGE_TEMPLATE_HEADERS)})" + ) + elif not prompt_path.is_file(): + problem = "a verdict without prompt.md" + else: + actual = template_version_of(prompt_path.read_bytes()) + if actual == recorded: + continue + found = "no template version" if actual is None else f"v{actual}" + problem = f"prompt.md is {found}, but its verdict records v{recorded}" + problems.append((case_dir.name, problem)) + return problems + + +def prompt_hash_problems(audit_dir: Path) -> list[tuple[str, str]]: + """Each judged case whose prompt.md is not the bytes its judge read. + + A runner records the sha256 of the bytes its judge read as the sidecar's + ``prompt_sha256`` (sidecars from before the runners did record none, and + are not checked). A verdict published after prompt.md was rewritten, say + by an audit-prepare racing the runner, keeps the hash of the old bytes, + so it disagrees. A verdict without prompt.md is + :func:`template_version_problems`' to report. Returns ``(case_id, + problem)`` for each case that fails, in case order. + """ + problems: list[tuple[str, str]] = [] + cases_root = audit_dir / "cases" + if not cases_root.is_dir(): + return problems + for case_dir in sorted(cases_root.iterdir()): + prompt_path = case_dir / "prompt.md" + if not ((case_dir / "verdict.json").is_file() and prompt_path.is_file()): + continue + recorded = (_sidecar(case_dir) or {}).get("prompt_sha256") + if recorded is None: + continue + if hashlib.sha256(prompt_path.read_bytes()).hexdigest() != recorded: + problems.append( + ( + case_dir.name, + "prompt.md is not the bytes its judge read (its sidecar's " + "prompt_sha256)", + ) + ) + return problems + + # --- Hedge detection ----------------------------------------------------------- # Published diagnoses must be definitive. These patterns catch the judge @@ -516,14 +661,18 @@ def _row_failure_source(meta: dict, model: str, requested_source: str) -> str: def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame]: """Fold verdicts into the annotation schema. - Returns ``{"row": ..., "case": ..., "missing": ..., "hedged": ...}``. + Returns ``{"row": ..., "case": ..., "missing": ..., "hedged": ..., + "template": ..., "prompt_hash": ...}``. ``row`` and ``case`` match the committed annotation CSV columns, extended with ``rationale`` and ``reference_suspect`` so the classifier's reasoning is preserved. ``missing`` lists cases whose verdict has not yet been produced (resumability). ``hedged`` lists case ids whose diagnosis or rationale hedges instead of diagnosing (see :data:`HEDGE_PATTERNS`) — delete those ``verdict.json`` files and re-run the classifier rather than - shipping them. + shipping them. ``template`` lists the judged cases whose prompt and + verdict disagree on the judge template version + (:func:`template_version_problems`), and ``prompt_hash`` those whose + prompt.md is not the bytes their judge read (:func:`prompt_hash_problems`). """ manifest = _load_manifest(audit_dir) cases_root = audit_dir / "cases" @@ -669,4 +818,10 @@ def collect_audit(country_dir: Path, audit_dir: Path) -> dict[str, pd.DataFrame] "case": pd.DataFrame(case_records, columns=case_columns), "missing": pd.DataFrame({"case_id": missing}, columns=["case_id"]), "hedged": pd.DataFrame({"case_id": hedged}, columns=["case_id"]), + "template": pd.DataFrame( + template_version_problems(audit_dir), columns=["case_id", "problem"] + ), + "prompt_hash": pd.DataFrame( + prompt_hash_problems(audit_dir), columns=["case_id", "problem"] + ), } diff --git a/policybench/cli.py b/policybench/cli.py index 94ffc572..f49e0ec4 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -15,6 +15,10 @@ PROGRAMS, get_programs, ) +from policybench.judge_template import ( + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, +) def _ensure_parent_dir(output_path: str) -> None: @@ -564,6 +568,16 @@ def main(): help="Optional CSV (scenario_id, variable, grounding) of authoritative " "engine facts rendered into matching case prompts", ) + audit_prepare_parser.add_argument( + "--template-version", + type=int, + required=True, + help="Judge template version for new and re-opened cases " + "(policybench.judge_template; a new audit uses the current one, " + f"{CURRENT_TEMPLATE_VERSION}). Required: a release reproduces its prompts " + "only on the version its verdicts were judged on. A judged case keeps " + "the version its verdict records", + ) audit_collect_parser = subparsers.add_parser( "audit-collect", @@ -1380,13 +1394,21 @@ def main(): for row in grounding_df.itertuples() if str(row.grounding).strip() } + template_version = args.template_version + if template_version not in JUDGE_TEMPLATE_HEADERS: + raise SystemExit( + f"audit-prepare: no judge template version {template_version}; " + f"the versions are {sorted(JUDGE_TEMPLATE_HEADERS)}" + ) cases = prepare_audit( Path(args.country_dir), Path(args.audit_dir), grounding_lookup=grounding_lookup, + template_version=template_version, ) print( - f"Prepared {len(cases)} audit cases under {args.audit_dir}. " + f"Prepared {len(cases)} audit cases under {args.audit_dir}; new and " + f"re-opened cases are on judge template v{template_version}. " f"Run scripts/run_audit_codex.sh {args.audit_dir} to classify." ) @@ -1395,6 +1417,20 @@ def main(): country_dir = Path(args.country_dir) out = collect_audit(country_dir, Path(args.audit_dir)) + template, prompt_hash = out["template"], out["prompt_hash"] + if not (template.empty and prompt_hash.empty): + # A verdict judged on other words than its prompt.md's is not + # this case's verdict. + for case_id, problem in template.itertuples(index=False): + print(f"TEMPLATE: {case_id}: {problem}") + for case_id, problem in prompt_hash.itertuples(index=False): + print(f"PROMPT: {case_id}: {problem}") + disagree = {*template["case_id"], *prompt_hash["case_id"]} + raise SystemExit( + f"{len(disagree)} verdicts disagree with their prompt.md on the " + "judge template version or the bytes their judge read; re-run " + "audit-prepare, which re-opens them, and re-judge" + ) hedged = out["hedged"] if not hedged.empty and not args.allow_hedged: for case_id in hedged["case_id"]: diff --git a/policybench/judge_template.py b/policybench/judge_template.py new file mode 100644 index 00000000..04d5d7f5 --- /dev/null +++ b/policybench/judge_template.py @@ -0,0 +1,159 @@ +"""The diagnosis judge's prompt template, by version. + +Every audit prompt (:func:`policybench.audit.render_case_prompt`) is one of +these headers followed by the case. A version's text never changes once a +verdict has been judged on it: a release carries a seed verdict forward only +when its case re-renders to the exact bytes the judge read, so the renderer +must still produce every version a carried verdict was judged on. Changing +the template means adding a version. + +- **v1**, the only template before versions were added, so every verdict + through release dashboard-data-20261010 (Claude Haiku 5.5), the last + release judged before versions existed. It tells the judge that the + reference pipeline "has survived an adversarial review program" and that + "the few real bugs found were fixed before this run". + That is not so: release dashboard-data-20261006's exclusion record stops + scoring 28 outputs for engine defects, and the 2026-10-05 reference + adversary confirmed four more. +- **v2**, v1 without that claim: the version a new audit names. + +Each verdict's provenance sidecar (``verdict.meta.json``) records the version +of the prompt it was judged on as ``judge_template_version``. A sidecar +without the field predates the versions and was judged on v1. A verdict +without a sidecar has no record of what its judge read, so it has no +recorded version: audit-prepare re-opens it and audit-collect refuses it. + +Version selection is explicit: every renderer +(:func:`policybench.audit.render_case_prompt`, +:func:`policybench.audit.prepare_audit`, ``policybench audit-prepare``) takes +the version as a required argument, and a judged case renders on the version +its sidecar records. Nothing picks a version by default, and nothing chooses +one by reading a prompt: :func:`template_version_of` only records what a +judge read and checks that a tree agrees with its sidecars. + +Dependency-free, so the audit runners can read a prompt's version. +""" + +from __future__ import annotations + +from collections.abc import Mapping + +# The sidecar field recording the version a verdict was judged on. +TEMPLATE_VERSION_FIELD = "judge_template_version" +# The version of a verdict whose sidecar does not record one. +UNRECORDED_TEMPLATE_VERSION = 1 +# The newest version: the one a new audit names. No renderer defaults to it. +CURRENT_TEMPLATE_VERSION = 2 + +_V1 = """\ +You are diagnosing why AI models missed a PolicyEngine reference value on a \ +US/UK tax-and-benefit estimation benchmark. The models answered from \ +parametric knowledge with no tools; PolicyEngine's microsimulation is the \ +reference. + +The reference value and its derivation are generated directly from the \ +engine's computation trace, and the reference pipeline has survived an \ +adversarial review program: every wrong-reference hypothesis raised by \ +earlier audits was adjudicated against primary sources, and the few real \ +bugs found were fixed before this run. Treat the reference and its \ +derivation as correct. Your job is NOT to re-litigate the reference — it is \ +to explain each model's mistake decisively. + +For every wrong model, write a `diagnosis`: 1-3 definitive sentences naming \ +the exact rule, eligibility pathway, deduction, threshold, or computation \ +step that model missed or misapplied, grounded in its own stated reasoning. \ +Be specific — "treated the 138% FPL MAGI limit as the only Medicaid pathway \ +and never applied the aged/disabled income test, which deducts the Medicare \ +Part B premium from countable income" — not generic ("got the income \ +calculation wrong"). If the model gave no usable reasoning, derive the \ +mistake from its answer: state what the correct derivation yields and what \ +shortcut the model's number is consistent with. + +Hedging is forbidden in `diagnosis` and `rationale`, and these phrasings \ +are mechanically rejected: "plausible"; "not enough evidence"; \ +"insufficient evidence/information"; "cannot/unable to \ +determine/verify/confirm/tell"; "difficult/hard to verify"; "may be/have"; \ +"might be/have"; "possibly"; "perhaps"; "unclear"; "the reference \ +is/appears/seems correct" or any other verdict on the reference. Write \ +definitively around them (e.g. "excess shelter costs are deductible", not \ +"may be deducted"). Doubt \ +about the reference belongs ONLY in reference_suspect + \ +reference_bug_hypothesis, and requires a concrete contradiction: the \ +derivation conflicts with a specific statute, regulation, or published \ +parameter you can name, or contradicts its own arithmetic. Absent that, set \ +reference_suspect=false and diagnose from the reference as ground truth. + +failure_source meanings: +- llm_error: the model reasoned or computed incorrectly (the usual case). +- prompt_ambiguity: the question is genuinely ambiguous; a careful expert \ +could read it more than one way. Name the two readings. +- reference_model_issue_fixed / reference_data_issue_fixed: the reference \ +value looks wrong (PolicyEngine logic / underlying data). Use with \ +reference_suspect=true and a concrete contradiction. +- parse_contract_failure: the model's answer was missing or unparseable, not a \ +substantive error. +- budget_exhausted_at_ceiling: the provider length-terminated every retry through \ +its maximum allowed completion budget. +- needs_review: genuinely cannot tell; name precisely what information is \ +missing. + +Output ONLY the JSON verdict matching the schema. Do not run any commands; all \ +information you need is below. +""" + +# The claim v2 drops: the clause ends v1's second paragraph's first sentence. +V1_REVIEW_CLAIM = ( + ", and the reference pipeline has survived an adversarial review program: " + "every wrong-reference hypothesis raised by earlier audits was adjudicated " + "against primary sources, and the few real bugs found were fixed before " + "this run" +) +if _V1.count(V1_REVIEW_CLAIM) != 1: + raise AssertionError("v1 states the review claim exactly once") +_V2 = _V1.replace(V1_REVIEW_CLAIM, "", 1) + +JUDGE_TEMPLATE_HEADERS: dict[int, str] = {1: _V1, 2: _V2} + + +def template_header(version: int) -> str: + """The header a prompt of ``version`` begins with.""" + if type(version) is not int or version not in JUDGE_TEMPLATE_HEADERS: + raise ValueError( + f"no judge template version {version!r}; the versions are " + f"{sorted(JUDGE_TEMPLATE_HEADERS)}" + ) + return JUDGE_TEMPLATE_HEADERS[version] + + +def template_version_of(prompt: str | bytes) -> int | None: + """The version whose header begins ``prompt``, or None if none does. + + No header is a prefix of another, so at most one matches. + """ + if isinstance(prompt, bytes): + try: + prompt = prompt.decode("utf-8") + except UnicodeDecodeError: + return None + for version, header in JUDGE_TEMPLATE_HEADERS.items(): + if prompt.startswith(header): + return version + return None + + +def recorded_template_version(meta: Mapping | None) -> int | None: + """The version a verdict's sidecar records it was judged on. + + A sidecar without the field means v1: it predates the versions. No + sidecar (None) records nothing, so it gives None, and so does a value + that names no version (null, a string, a boolean, an unknown number): + the verdict's template is unknown. + """ + if meta is None: + return None + if TEMPLATE_VERSION_FIELD not in meta: + return UNRECORDED_TEMPLATE_VERSION + value = meta[TEMPLATE_VERSION_FIELD] + if type(value) is int and value in JUDGE_TEMPLATE_HEADERS: + return value + return None diff --git a/scripts/finish_adds0928.py b/scripts/finish_adds0928.py index 1ea0cbb3..db7b2b8f 100644 --- a/scripts/finish_adds0928.py +++ b/scripts/finish_adds0928.py @@ -47,6 +47,11 @@ "us_adjudications.json", ) JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on. Its release, RELEASE_TAG, judged every verdict on +# v1, so re-running the driver reproduces its prompts. A seed case keeps the +# version its sidecar records. +JUDGE_TEMPLATE_VERSION = 1 # main at release dashboard-data-20260922c, whose committed references are the base. BASE_COMMIT = "3220a7a62b6be83032e9313c9df539c619ad8932" BASE_REFERENCE_SHA256 = { @@ -492,7 +497,12 @@ def prepare_cases(args, bundle) -> None: (str(r.scenario_id), str(r.variable)): str(r.grounding) for r in grounding.itertuples() } - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) pending = validate_verdicts(audit, remove_invalid=True) write_json(args.stage_dir / "pending.json", pending) print(f"Prepared audit: {len(pending)} cases need Opus 5.5") diff --git a/scripts/finish_gpt61sol.py b/scripts/finish_gpt61sol.py index 11551e56..982ae6f6 100644 --- a/scripts/finish_gpt61sol.py +++ b/scripts/finish_gpt61sol.py @@ -53,6 +53,11 @@ "us_adjudications.json", ) JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on. Its release, RELEASE_TAG, judged every verdict on +# v1, so re-running the driver reproduces its prompts. A seed case keeps the +# version its sidecar records. +JUDGE_TEMPLATE_VERSION = 1 ADJUDICATIONS = "us_adjudications.json" # Stage files export binds: the cases GPT-6.1 Sol re-opened, and the # wording-only amendments a developer lists for them. @@ -1414,7 +1419,12 @@ def prepare_cases(args, bundle) -> dict[str, dict[str, str]]: (str(r.scenario_id), str(r.variable)): str(r.grounding) for r in grounding.itertuples() } - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) changes = check_prompt_changes(audit, seeded) write_json( args.stage_dir / "prompt-changes.json", diff --git a/scripts/finish_haiku55.py b/scripts/finish_haiku55.py index d8b315b5..233f09ec 100644 --- a/scripts/finish_haiku55.py +++ b/scripts/finish_haiku55.py @@ -69,6 +69,11 @@ "us_adjudications.json", ) JUDGE_MODEL = "claude-opus-5-5" +# The judge template (policybench.judge_template) this driver renders new +# and re-opened cases on. Its release, RELEASE_TAG, judged every verdict on +# v1, so re-running the driver reproduces its prompts. A seed case keeps the +# version its sidecar records. +JUDGE_TEMPLATE_VERSION = 1 ADJUDICATIONS = "us_adjudications.json" # Stage files export binds: the cases Claude Haiku 5.5 re-opened, and the # wording-only amendments a developer lists for them. @@ -2334,7 +2339,12 @@ def prepare_cases(args, bundle) -> dict[str, dict[str, str]]: if (source / name).is_file(): shutil.copyfile(source / name, target / name) lookup = grounding_lookup(args.grounding) - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) changes = check_prompt_changes(audit, seeded) write_json( args.stage_dir / "prompt-changes.json", @@ -4327,6 +4337,9 @@ def render_audit_with( A scratch run directory links every other bundle file and holds the build's, so nothing in the stage changes. The order is prepare_audit's. + Every prompt is on JUDGE_TEMPLATE_VERSION, the version of every verdict + this release's stage holds; install-references refuses a stage whose + prepare_audit rendering differs. """ import tempfile @@ -4350,7 +4363,11 @@ def render_audit_with( with object_strings(): cases = build_audit_cases(run / "us", grounding_lookup=lookup) return [ - (case.case_id, render_case_prompt(case), case.to_manifest_row()) + ( + case.case_id, + render_case_prompt(case, template_version=JUDGE_TEMPLATE_VERSION), + case.to_manifest_row(), + ) for case in cases ] @@ -4545,7 +4562,12 @@ def install_references(args) -> None: for case, reason in reopen.items(): set_aside(audit, case, reason) with object_strings(): - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit( + bundle / "us", + audit, + grounding_lookup=lookup, + template_version=JUDGE_TEMPLATE_VERSION, + ) manifest = [ json.loads(line) for line in (audit / "cases.jsonl").read_text().splitlines() ] diff --git a/scripts/run_audit_claude.sh b/scripts/run_audit_claude.sh index 490a7cc7..a07b550d 100755 --- a/scripts/run_audit_claude.sh +++ b/scripts/run_audit_claude.sh @@ -28,7 +28,10 @@ # an advisor model; and unless the judge's one accepted StructuredOutput # call (any other is one the schema refused) answered exactly the verdict; # - it reads a private copy of prompt.md, hashed before it runs; the sidecar -# records that hash. A verdict is rejected if prompt.md no longer has it +# records that hash and the judge template version the copy begins with +# (judge_template_version, policybench.judge_template; null for a prompt +# audit-prepare did not render). A verdict is rejected if prompt.md no +# longer has it # at extraction, or if the transcript's user events are not exactly one # text message equal to the judged prompt (plus Claude Code's own # StructuredOutput nudge, PROMPT_NUDGE) and the results of the judge's @@ -354,13 +357,18 @@ extract_verdict() { case_dir="$1"; out_tmp="$2"; meta_tmp="$3"; judged="$4"; judged_sha="$5" "$PYTHON" - "$case_dir" "$out_tmp" "$meta_tmp" "$MODEL" "$CLI_VERSION" \ "$CONFIG_DIR" "$AUTH" "$DECLARED" "$DISALLOWED" "$ATTACHMENTS" \ - "$EFFORT" "$judged" "$judged_sha" "$PROMPT_NUDGE" "$EVENT_TYPES" <<'PY' + "$EFFORT" "$judged" "$judged_sha" "$PROMPT_NUDGE" "$EVENT_TYPES" \ + "$SCRIPT_DIR" <<'PY' import datetime, glob, hashlib, json, re, shutil, sys from pathlib import Path (case_dir, out_path, meta_path, requested_model, cli_version, config_dir, auth, declared, disallowed, attachments, effort, judged_path, judged_sha, - nudge, event_types) = sys.argv[1:16] + nudge, event_types, script_dir) = sys.argv[1:17] +# The judge template versions, from this runner's own checkout. +sys.path.insert(0, str(Path(script_dir).resolve().parent)) +from policybench.judge_template import template_version_of + case = Path(case_dir) try: envelope = json.load(open(case / "claude.json")) @@ -597,6 +605,9 @@ meta = { "verdict_sha256": hashlib.sha256(verdict_bytes).hexdigest(), # The exact bytes the judge read on stdin, hashed before it ran. "prompt_sha256": judged_sha, + # The judge template those bytes begin with (policybench.judge_template), + # or null for a prompt audit-prepare did not render. + "judge_template_version": template_version_of(judged), "judge_model_requested": requested_model, "judge_model_reported": sorted((envelope.get("modelUsage") or {}).keys()), "judge_cli_version": cli_version, @@ -631,8 +642,11 @@ classify_one() { [ -f "$prompt" ] || return 0 case_ok "$out" "$name" 2>/dev/null && return 0 # No valid verdict: any sidecar left behind describes a verdict that no - # longer exists (re-prepared case) and must not outlive it. - rm -f "$tmp" "$meta_tmp" "$envelope" "$case_dir/verdict.meta.json" \ + # longer exists (re-prepared case) and must not outlive it. An invalid + # verdict.json goes too, so no verdict sits beside this run's sidecar until + # this run's verdict is published: audit-prepare removes a sidecar only + # together with the verdict it describes. + rm -f "$tmp" "$meta_tmp" "$envelope" "$out" "$case_dir/verdict.meta.json" \ "$case_dir/claude.transcript.jsonl" work=$(mktemp -d "${TMPDIR:-/tmp}/pb-judge.XXXXXX") || { echo "[FAIL] $(basename "$case_dir") (no scratch directory)" @@ -680,8 +694,12 @@ classify_one() { rm -f "$judged" if [ "$extracted" = 0 ] && verdict_ok "$tmp"; then if problem=$(case_ok "$tmp" "$name" 2>&1); then - mv -f "$tmp" "$out" + # The sidecar first: an interrupted publish leaves at most a sidecar + # without a verdict, which the next run removes, and never a verdict + # without its sidecar, which audit-prepare would read as a legacy v1 + # verdict with no record of the bytes its judge read. mv -f "$meta_tmp" "$case_dir/verdict.meta.json" + mv -f "$tmp" "$out" echo "[ok] $name" else # Answered, but not for this case's models: pending, and the run goes on. diff --git a/scripts/run_audit_codex.sh b/scripts/run_audit_codex.sh index dfb7d623..b2a60dfa 100755 --- a/scripts/run_audit_codex.sh +++ b/scripts/run_audit_codex.sh @@ -12,9 +12,13 @@ # # Judge provenance: beside each verdict.json the runner writes verdict.meta.json # (the same sidecar scripts/run_audit_claude.sh writes) with the model Codex -# reports in its log header, the model requested, the UTC timestamp, and the +# reports in its log header, the model requested, the UTC timestamp, the # verdict's sha256, so a re-judged case can never keep the other runner's -# provenance. +# provenance, and the bytes Codex judged: a private copy of prompt.md, taken +# before it runs, whose sha256 the sidecar records as prompt_sha256 and whose +# judge template version it records as judge_template_version +# (policybench.judge_template; null for a prompt audit-prepare did not render). +# A verdict is not published if prompt.md no longer holds those bytes. set -u SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" @@ -72,8 +76,23 @@ classify_one() { # temp file and publish atomically only once it validates, so an interrupted # run never leaves a half-written verdict that looks done. Default reasoning # effort (xhigh) is wasteful for classification, so it is lowered. A sidecar - # left from a previous verdict describes a verdict that no longer exists. - rm -f "$tmp" "$case_dir/verdict.meta.json" + # left from a previous verdict describes a verdict that no longer exists, + # and an invalid verdict.json goes too, so no verdict sits beside this + # run's sidecar until this run's verdict is published: audit-prepare + # removes a sidecar only together with the verdict it describes. + rm -f "$tmp" "$out" "$case_dir/verdict.meta.json" + # Codex reads a private copy, so the sidecar describes the bytes it judged + # even if audit-prepare rewrites prompt.md meanwhile. + judged=$(mktemp "${TMPDIR:-/tmp}/pb-codex-prompt.XXXXXX") || { + echo "[FAIL] $(basename "$case_dir") (cannot copy prompt.md)"; return 0; } + # Each case runs in its own background subshell: a worker terminated + # mid-case still removes its copy. + trap 'rm -f "$judged"' EXIT + trap 'rm -f "$judged"; exit 143' TERM + trap 'rm -f "$judged"; exit 130' INT + cp "$prompt" "$judged" || { + rm -f "$judged"; echo "[FAIL] $(basename "$case_dir") (cannot copy prompt.md)" + return 0; } codex exec \ --sandbox read-only \ --skip-git-repo-check \ @@ -83,26 +102,48 @@ classify_one() { -c model_reasoning_effort="$EFFORT" \ --output-schema "$SCHEMA" \ -o "$tmp" \ - - < "$prompt" > "$case_dir/codex.log" 2>&1 + - < "$judged" > "$case_dir/codex.log" 2>&1 if verdict_ok "$tmp" && write_provenance "$tmp" "$case_dir/codex.log" \ - "$case_dir/verdict.meta.json.tmp"; then - mv -f "$tmp" "$out" + "$case_dir/verdict.meta.json.tmp" "$judged" "$prompt"; then + # The sidecar first: an interrupted publish leaves at most a sidecar + # without a verdict, which the next run removes, and never a verdict + # without its sidecar, which audit-prepare would read as a legacy v1 + # verdict with no record of the bytes its judge read. mv -f "$case_dir/verdict.meta.json.tmp" "$case_dir/verdict.meta.json" + mv -f "$tmp" "$out" echo "[ok] $(basename "$case_dir")" else rm -f "$tmp" "$case_dir/verdict.meta.json.tmp" echo "[FAIL] $(basename "$case_dir") (see codex.log)" fi + rm -f "$judged" } # Provenance sidecar for a validated verdict: the model Codex reports in its # log header (`model: ...`), the model requested (or "default"), the UTC time, -# and the verdict's sha256 so the sidecar cannot outlive the verdict it describes. +# the verdict's sha256 so the sidecar cannot outlive the verdict it describes, +# and the sha256 and judge template version of the copy Codex judged. Fails, +# so the verdict is not published, when prompt.md no longer holds that copy's +# bytes. write_provenance() { - verdict_path="$1"; log_path="$2"; meta_out="$3" - "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" <<'PY' + verdict_path="$1"; log_path="$2"; meta_out="$3"; judged_path="$4" + prompt_path="$5" + "$PYTHON" - "$verdict_path" "$log_path" "$meta_out" "${AUDIT_MODEL:-default}" \ + "$judged_path" "$prompt_path" "$SCRIPT_DIR" <<'PY' import datetime, hashlib, json, re, sys -verdict_path, log_path, meta_out, requested = sys.argv[1:5] +from pathlib import Path +(verdict_path, log_path, meta_out, requested, judged_path, prompt_path, + script_dir) = sys.argv[1:8] +sys.path.insert(0, str(Path(script_dir).resolve().parent)) +from policybench.judge_template import template_version_of +judged = open(judged_path, "rb").read() +try: + live = open(prompt_path, "rb").read() +except OSError: + live = None +if live != judged: + print("prompt.md changed while Codex judged it", file=sys.stderr) + sys.exit(1) verdict = open(verdict_path, "rb").read() reported = [] try: @@ -117,6 +158,9 @@ meta = { "judge_model_requested": requested, "judge_model_reported": reported, "judged_at_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + # The exact bytes Codex read on stdin, copied before it ran. + "prompt_sha256": hashlib.sha256(judged).hexdigest(), + "judge_template_version": template_version_of(judged), } json.dump(meta, open(meta_out, "w"), indent=2, sort_keys=True) PY diff --git a/tests/test_audit.py b/tests/test_audit.py index 3d4a3d76..a52b2a67 100644 --- a/tests/test_audit.py +++ b/tests/test_audit.py @@ -22,6 +22,7 @@ prepare_audit, render_case_prompt, ) +from policybench.judge_template import CURRENT_TEMPLATE_VERSION @pytest.fixture @@ -87,7 +88,7 @@ def test_build_audit_cases_groups_wrong_by_case(country_dir: Path): def test_render_case_prompt_includes_reference_and_models(country_dir: Path): case = next(c for c in build_audit_cases(country_dir) if c.scenario_id == "s0") - prompt = render_case_prompt(case) + prompt = render_case_prompt(case, template_version=CURRENT_TEMPLATE_VERSION) assert "POLICYENGINE REFERENCE VALUE: $0.00" in prompt assert "m1: answered $250.00" in prompt assert "Used gross income test." in prompt @@ -114,7 +115,9 @@ def test_parse_verdict_tolerates_prose_around_json(tmp_path: Path): def test_prepare_audit_writes_layout(country_dir: Path, tmp_path: Path): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) assert (audit_dir / "schema.json").exists() assert (audit_dir / "cases.jsonl").exists() for case in cases: @@ -127,7 +130,9 @@ def test_collect_audit_folds_verdicts_and_tracks_missing( country_dir: Path, tmp_path: Path ): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) # Write a verdict for the s0 case only; leave s1 missing. s0 = next(c for c in cases if c.scenario_id == "s0") verdict = { @@ -169,7 +174,9 @@ def test_collect_audit_rejects_invalid_failure_source( country_dir: Path, tmp_path: Path ): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") (audit_dir / "cases" / s0.case_id / "verdict.json").write_text( json.dumps( @@ -223,7 +230,9 @@ def test_reference_derivation_reaches_the_prompt(tmp_path: Path): case = build_audit_cases(d)[0] assert "over the income test" in case.reference_derivation - assert "HOW POLICYENGINE DERIVED THE REFERENCE" in render_case_prompt(case) + assert "HOW POLICYENGINE DERIVED THE REFERENCE" in render_case_prompt( + case, template_version=CURRENT_TEMPLATE_VERSION + ) def test_binary_variable_renders_yes_no(tmp_path: Path): @@ -286,7 +295,9 @@ def test_missing_prediction_renders_as_missing(tmp_path: Path): case = next(c for c in build_audit_cases(d) if c.scenario_id == "s0") silent = next(m for m in case.wrong_models if m.model == "silent") assert silent.prediction == "missing" - assert "answered missing" in render_case_prompt(case) + assert "answered missing" in render_case_prompt( + case, template_version=CURRENT_TEMPLATE_VERSION + ) def test_collect_defaults_missing_model_to_case_source( @@ -294,7 +305,9 @@ def test_collect_defaults_missing_model_to_case_source( ): """When the verdict omits a wrong model, its row inherits the case source.""" audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") (audit_dir / "cases" / s0.case_id / "verdict.json").write_text( json.dumps( @@ -372,7 +385,7 @@ def test_collect_empty_wrong_set_keeps_header(tmp_path: Path): ] ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) out = collect_audit(d, audit_dir) assert list(out["row"].columns) and "failure_source" in out["row"].columns assert list(out["case"].columns) and "case_failure_source" in out["case"].columns @@ -410,7 +423,7 @@ def test_parse_failure_only_case_skips_codex_and_is_deterministic(tmp_path: Path ] ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - cases = prepare_audit(d, audit_dir) + cases = prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) s0 = next(c for c in cases if c.scenario_id == "s0") # No prompt is written for a parse-failure-only case (no Codex call). assert not (audit_dir / "cases" / s0.case_id / "prompt.md").exists() @@ -446,7 +459,7 @@ def test_budget_exhaustion_source_survives_deterministic_missing_audit( ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) out = collect_audit(d, audit_dir) assert out["row"].iloc[0]["failure_source"] == "budget_exhausted_at_ceiling" @@ -494,7 +507,7 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): ] ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - cases = prepare_audit(d, audit_dir) + cases = prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) s0 = cases[0] verdict_path = audit_dir / "cases" / s0.case_id / "verdict.json" verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') @@ -514,14 +527,22 @@ def test_reprepare_drops_stale_verdict_when_case_changed(tmp_path: Path): } ] ).to_csv(d / "predictions.csv", index=False) - prepare_audit(d, audit_dir) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) assert not verdict_path.exists() # stale verdict dropped assert not meta_path.exists() # and its provenance sidecar with it - # An unchanged case keeps its verdict. + # An unchanged case keeps its verdict, judged on the prompt's template + # (the re-opened case's prompt is the current version's). verdict_path.write_text('{"case_failure_source": "llm_error", "models": []}') - prepare_audit(d, audit_dir) + meta_path.write_text(json.dumps({"judge_template_version": 2})) + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) assert verdict_path.exists() + # A verdict whose sidecar records no version was judged on v1, so the + # current version's prompt is not the one it was judged on. + meta_path.write_text('{"judge_model_requested": "opus"}') + prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) + assert not verdict_path.exists() + assert not meta_path.exists() def test_collect_audit_coerces_parse_source_for_parsed_predictions(tmp_path: Path): @@ -545,7 +566,7 @@ def test_collect_audit_coerces_parse_source_for_parsed_predictions(tmp_path: Pat ).to_csv(d / "predictions.csv", index=False) audit_dir = tmp_path / "audit" - cases = prepare_audit(d, audit_dir) + cases = prepare_audit(d, audit_dir, template_version=CURRENT_TEMPLATE_VERSION) case = cases[0] verdict_path = audit_dir / "cases" / case.case_id / "verdict.json" verdict_path.write_text( @@ -611,7 +632,9 @@ def test_collect_uses_per_model_diagnosis_for_row_annotation( country_dir: Path, tmp_path: Path ): audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") verdict = make_verdict( "Applied the net-income allotment formula without the earned income " @@ -637,7 +660,9 @@ def test_collect_flags_hedged_verdicts_for_rejudging(country_dir: Path, tmp_path # The shipped 20260707c failure class: the judge answers "is the # reference wrong?" instead of diagnosing the model. audit_dir = tmp_path / "audit" - cases = prepare_audit(country_dir, audit_dir) + cases = prepare_audit( + country_dir, audit_dir, template_version=CURRENT_TEMPLATE_VERSION + ) s0 = next(c for c in cases if c.scenario_id == "s0") verdict = make_verdict( "The reference is plausible under the stated PolicyEngine " @@ -676,7 +701,7 @@ def test_prompt_treats_reference_as_verified_and_demands_diagnosis( country_dir: Path, ): case = next(c for c in build_audit_cases(country_dir) if c.scenario_id == "s0") - prompt = render_case_prompt(case) + prompt = render_case_prompt(case, template_version=CURRENT_TEMPLATE_VERSION) assert "Treat the reference and its derivation as correct" in prompt assert "diagnosis" in prompt assert "mechanically rejected" in prompt @@ -688,9 +713,15 @@ def test_grounding_lookup_renders_engine_facts_block(country_dir: Path): cases = build_audit_cases(country_dir, grounding_lookup=lookup) grounded = next(c for c in cases if c.scenario_id == "s0") bare = next(c for c in cases if c.scenario_id == "s1") - assert "AUTHORITATIVE ENGINE FACTS" in render_case_prompt(grounded) - assert "SENIOR_OR_DISABLED" in render_case_prompt(grounded) - assert "AUTHORITATIVE ENGINE FACTS" not in render_case_prompt(bare) + assert "AUTHORITATIVE ENGINE FACTS" in render_case_prompt( + grounded, template_version=CURRENT_TEMPLATE_VERSION + ) + assert "SENIOR_OR_DISABLED" in render_case_prompt( + grounded, template_version=CURRENT_TEMPLATE_VERSION + ) + assert "AUTHORITATIVE ENGINE FACTS" not in render_case_prompt( + bare, template_version=CURRENT_TEMPLATE_VERSION + ) def test_schema_requires_per_model_diagnosis(): diff --git a/tests/test_finish_gpt61sol.py b/tests/test_finish_gpt61sol.py index 6485d55e..2ed39158 100644 --- a/tests/test_finish_gpt61sol.py +++ b/tests/test_finish_gpt61sol.py @@ -28,7 +28,16 @@ import finish_gpt61sol as driver # noqa: E402 -from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 +from policybench.audit import ( # noqa: E402 + AUDIT_OUTPUT_SCHEMA, + template_version_problems, +) +from policybench.judge_template import ( # noqa: E402 + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, + V1_REVIEW_CLAIM, + template_version_of, +) NEW = "gpt-6.1-sol" SLUG = "gpt61sol" @@ -966,7 +975,9 @@ def seeded_stage(tmp_path, monkeypatch): lookup = {("s1", "snap"): "Gross test: pass."} seed = tmp_path / "seed" seed_board = _board(tmp_path / "seed-board", INCUMBENT_ROWS) - prepare_audit(seed_board / "us", seed, grounding_lookup=lookup) + # Release 20260929's seed was judged on judge template v1, before versions + # existed, so its sidecars record none. + prepare_audit(seed_board / "us", seed, grounding_lookup=lookup, template_version=1) for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): write_verdict( seed / "cases" / item["case_id"], @@ -996,9 +1007,16 @@ def prepare(new_rows, *, grounding_path=grounding): return seed, stage, prepare +@pytest.mark.parametrize( + "version", sorted({driver.JUDGE_TEMPLATE_VERSION, CURRENT_TEMPLATE_VERSION}) +) def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( - seeded_stage, + seeded_stage, monkeypatch, version ): + """The driver re-opens the case the new model joins on the version it + names: v1 as committed, or v2 as a new release's driver names it. The + other case keeps the seed's bytes on the v1 template it was judged on.""" + monkeypatch.setattr(driver, "JUDGE_TEMPLATE_VERSION", version) seed, stage, prepare = seeded_stage audit = prepare( [ @@ -1015,16 +1033,28 @@ def test_a_case_the_new_model_joins_is_rejudged_and_the_rest_carry_over( assert (audit / "cases" / untouched / name).read_bytes() == ( seed / "cases" / untouched / name ).read_bytes() + assert ( + template_version_of((audit / "cases" / untouched / "prompt.md").read_bytes()) + == 1 + ) + assert ( + template_version_of((audit / "cases" / joined / "prompt.md").read_bytes()) + == version + ) assert json.loads((stage / "prompt-changes.json").read_text()) == { "added": [], "changed": [joined], "kept": [untouched], } wrong = ["m1", "m2", NEW] - write_verdict(audit / "cases" / joined, _verdict(wrong)) + write_verdict( + audit / "cases" / joined, _verdict(wrong), judge_template_version=version + ) seed_binding = driver.load_seed(stage) assert seed_binding == driver.seed_digest(seed) assert driver.validate_verdicts(audit, seed=seed_binding) == [] + # The stage validates; on v2 it mixes a v1 seed and a v2 verdict. + assert template_version_problems(audit) == [] # GPT-6.1 Sol misses s0 alone: s0 is re-opened, s1 carries over. @@ -1373,54 +1403,139 @@ def test_each_20260929_entry_names_its_seed_verdict_as_its_sidecar_records_it(): assert named_item(entry) == replaced_item(found, key in waved), entry -@pytest.mark.slow -def test_every_seed_prompt_rerenders_from_the_committed_snapshot(tmp_path): - """Release 20260929's 45-model board, as BASE_COMMIT holds it, renders - exactly the seed's cases and prompts. - - So check_prompt_changes may attribute every changed or new prompt in a - stage to GPT-6.1 Sol joining its case. #182's review excluded - scenario_023 head_medicaid_eligible and rewrote its adjudication and case - note; neither enters a prompt, and its prompt is unchanged. - """ - if not (SEED.is_dir() and GROUNDING.is_file()): - pytest.skip("release 20260929's audit or grounding is not on this machine") - from policybench.audit import prepare_audit - - bundle = tmp_path / "publish" / driver.RUN_NAME +def _committed_bundle(root: Path, blob) -> Path: + """The board a release committed, read through ``blob`` (a function from + a repository path to its bytes at that release's commit), as a bundle + prepare_audit can render: ``root/publish//us``.""" + bundle = root / "publish" / driver.RUN_NAME (bundle / "us").mkdir(parents=True) - # From git: after this release's freeze the working tree holds 20260930. for name in (*driver.REFERENCE_FILES, "predictions.csv.gz"): - (bundle / "us" / name).write_bytes( - driver.base_commit_blob((driver.SNAPSHOT / name).relative_to(driver.ROOT)) - ) + (bundle / "us" / name).write_bytes(blob(SNAPSHOT_PATH / name)) (bundle / "annotations").mkdir() for name in driver.ANNOTATION_FILES: (bundle / "annotations" / name).write_bytes( - driver.base_commit_blob( - (driver.ANNOTATIONS / name).relative_to(driver.ROOT) - ) + blob(Path("annotations") / driver.RUN_NAME / name) ) + return bundle / "us" + + +def _prepare_like_the_driver(country_dir: Path, audit: Path, **kwargs) -> None: + """prepare_audit with the grounding and the object strings the driver + renders with (resolve_base sets them).""" + from policybench.audit import prepare_audit + grounding = pd.read_csv(GROUNDING) lookup = { (str(r.scenario_id), str(r.variable)): str(r.grounding) for r in grounding.itertuples() } - audit = tmp_path / "audit" - # prepare renders with object strings, as resolve_base sets them. arrow = hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string") with ( pd.option_context("future.infer_string", False) if arrow else contextlib.nullcontext() ): - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + prepare_audit(country_dir, audit, grounding_lookup=lookup, **kwargs) + + +@pytest.mark.slow +def test_every_seed_prompt_rerenders_from_the_committed_snapshot(tmp_path): + """Release 20260929's 45-model board, as BASE_COMMIT holds it, renders + exactly the seed's cases and prompts on judge template v1, the template + the seed was judged on. + + So check_prompt_changes may attribute every changed or new prompt in a + stage to GPT-6.1 Sol joining its case. #182's review excluded + scenario_023 head_medicaid_eligible and rewrote its adjudication and case + note; neither enters a prompt, and its prompt is unchanged. On v2 every + prompt is the seed's without v1's review claim. + """ + if not (SEED.is_dir() and GROUNDING.is_file()): + pytest.skip("release 20260929's audit or grounding is not on this machine") + # From git: after this release's freeze the working tree holds 20260930. + country_dir = _committed_bundle(tmp_path, driver.base_commit_blob) + audit = tmp_path / "audit" + _prepare_like_the_driver(country_dir, audit, template_version=1) assert (audit / "cases.jsonl").read_bytes() == (SEED / "cases.jsonl").read_bytes() rendered = driver.seed_prompt_digests(audit) assert rendered == driver.seed_prompt_digests(SEED) and len(rendered) == 674 case = "us__scenario_023__head_medicaid_eligible" meta = json.loads((SEED / "cases" / case / "verdict.meta.json").read_text()) assert rendered[case] == meta["prompt_sha256"] + current = tmp_path / "current" + _prepare_like_the_driver(country_dir, current, template_version=2) + for case_dir in sorted((SEED / "cases").iterdir()): + seed_prompt = (case_dir / "prompt.md").read_text() + assert (current / "cases" / case_dir.name / "prompt.md").read_text() == ( + seed_prompt.replace(V1_REVIEW_CLAIM, "", 1) + ), case_dir.name + + +# The audit trees releases froze (git-ignored, local only), each with the +# commit holding the board it was rendered from: release 20260929's (the +# GPT-6.1 Sol seed, BASE_COMMIT) and release 20260930's (RELEASE_COMMIT), +# which release dashboard-data-20261006 carried over unchanged. +FROZEN_AUDITS = { + "20260929": (SEED, "base"), + "20260930": ( + Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/gpt61sol-stage/" + "gpt61sol-v1/audit" + ), + "release", + ), +} + + +@pytest.mark.slow +@pytest.mark.parametrize("release", sorted(FROZEN_AUDITS)) +def test_every_frozen_verdict_rerenders_on_its_recorded_template(tmp_path, release): + """Local only: copied into a stage as prepare_cases copies a seed, every + verdict of a frozen audit tree re-renders on the template version its + sidecar records (absent: v1) to the exact prompt it was judged on, so + prepare_audit keeps every prompt, verdict and sidecar byte for byte + whichever version its caller names; and the tree validates.""" + from policybench.audit import template_version_problems + from policybench.judge_template import recorded_template_version + + tree, commit = FROZEN_AUDITS[release] + if not (tree.is_dir() and GROUNDING.is_file()): + pytest.skip(f"release {release}'s audit or grounding is not on this machine") + blob = driver.base_commit_blob if commit == "base" else release_blob + country_dir = _committed_bundle(tmp_path, blob) + stage = tmp_path / "stage" + judged = {} + for case_dir in sorted((tree / "cases").iterdir()): + if not (case_dir / "verdict.json").is_file(): + continue + target = stage / "cases" / case_dir.name + target.mkdir(parents=True) + judged[case_dir.name] = {} + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + if (case_dir / name).is_file(): + data = (case_dir / name).read_bytes() + (target / name).write_bytes(data) + judged[case_dir.name][name] = data + assert len(judged) == 674 + assert template_version_problems(stage) == [] + versions = { + case: recorded_template_version( + json.loads(files["verdict.meta.json"]) + if "verdict.meta.json" in files + else None + ) + for case, files in judged.items() + } + assert set(versions.values()) == {1} + for version in sorted(JUDGE_TEMPLATE_HEADERS): + _prepare_like_the_driver(country_dir, stage, template_version=version) + cases_jsonl = (stage / "cases.jsonl").read_bytes() + assert cases_jsonl == (tree / "cases.jsonl").read_bytes(), version + for case, files in judged.items(): + for name, data in files.items(): + path = stage / "cases" / case / name + assert path.read_bytes() == data, (version, case, name) + assert template_version_problems(stage) == [] def test_check_prompt_changes_names_new_incumbent_only_cases(tmp_path): diff --git a/tests/test_finish_haiku55.py b/tests/test_finish_haiku55.py index 8d77f211..4f798b1f 100644 --- a/tests/test_finish_haiku55.py +++ b/tests/test_finish_haiku55.py @@ -1148,7 +1148,9 @@ def seeded_stage(tmp_path, monkeypatch): lookup = {("s1", "snap"): "Gross test: pass."} seed = tmp_path / "seed" seed_board = _board(tmp_path / "seed-board", INCUMBENT_ROWS) - prepare_audit(seed_board / "us", seed, grounding_lookup=lookup) + # Release 20261006's seed was judged on judge template v1, before versions + # existed, so its sidecars record none. + prepare_audit(seed_board / "us", seed, grounding_lookup=lookup, template_version=1) for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): write_verdict( seed / "cases" / item["case_id"], @@ -1965,7 +1967,8 @@ def test_each_20261006_entry_names_its_seed_verdict_as_its_sidecar_records_it(): @pytest.mark.slow def test_every_seed_prompt_rerenders_except_the_nine_reworded(tmp_path): """Local only: release 20261006's 46-model board, as BASE_COMMIT holds it, - renders exactly the seed's cases, and every seed prompt except the nine + renders exactly the seed's cases, and on judge template v1, the template + the seed was judged on, every seed prompt except the nine REWORDED_SINCE_SEED, whose reference explanations it reworded (this reads the 43 MB predictions from git into scratch). So check_prompt_changes may attribute every other changed or new prompt to @@ -1996,7 +1999,8 @@ def test_every_seed_prompt_rerenders_except_the_nine_reworded(tmp_path): if arrow else contextlib.nullcontext() ): - prepare_audit(bundle / "us", audit, grounding_lookup=lookup) + # The seed was judged on judge template v1. + prepare_audit(bundle / "us", audit, grounding_lookup=lookup, template_version=1) # The manifest records each case's reference derivation, so it differs # from the seed's in that field of the nine cases and nowhere else. ours, seeds = ( @@ -6140,8 +6144,11 @@ def patch(name, value): lookup = {("s1", "snap"): "Gross test: pass."} seed = root / "seed" seed_board = _board(root / "seed-board", INCUMBENT_ROWS) + # The seed is on judge template v1, as release 20261006's was. with driver.object_strings(): - prepare_audit(seed_board / "us", seed, grounding_lookup=lookup) + prepare_audit( + seed_board / "us", seed, grounding_lookup=lookup, template_version=1 + ) for item in map(json.loads, (seed / "cases.jsonl").read_text().splitlines()): write_verdict( seed / "cases" / item["case_id"], diff --git a/tests/test_judge_provenance.py b/tests/test_judge_provenance.py index 00a6722d..8e891054 100644 --- a/tests/test_judge_provenance.py +++ b/tests/test_judge_provenance.py @@ -21,7 +21,12 @@ verify_adjudications_keep_judge_verdicts, ) -from policybench.audit import AUDIT_OUTPUT_SCHEMA # noqa: E402 +from policybench.audit import ( # noqa: E402 + AUDIT_OUTPUT_SCHEMA, + prompt_hash_problems, + template_version_problems, +) +from policybench.judge_template import template_header # noqa: E402 VERDICT = { "reference_suspect": False, @@ -289,6 +294,13 @@ def test_rejudging_through_the_other_runner_replaces_provenance(tmp_path: Path): meta = json.loads((case_dir / "verdict.meta.json").read_text()) assert meta["judge_runner"] == "scripts/run_audit_codex.sh" assert meta["judge_model_reported"] == ["gpt-5.6-sol"] + # The fixture's prompt is no judge template's. + assert meta["judge_template_version"] is None + # The bytes Codex judged, from its private copy of prompt.md. + assert ( + meta["prompt_sha256"] + == hashlib.sha256((case_dir / "prompt.md").read_bytes()).hexdigest() + ) assert ( meta["verdict_sha256"] == hashlib.sha256((case_dir / "verdict.json").read_bytes()).hexdigest() @@ -508,3 +520,213 @@ def test_adjudication_flag_matches_the_verdict_or_names_its_run(tmp_path: Path): [dict(entry, scenario_id="scenario_002", judge_reference_suspect=False)], cases, ) + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_both_runners_record_the_judge_template_version(tmp_path: Path): + """Each runner's sidecar records the template version of the prompt its + verdict was judged on, so prepare_audit can render the case on it again.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + for runner, version in (("run_audit_claude.sh", 2), ("run_audit_codex.sh", 1)): + (case_dir / "verdict.json").unlink(missing_ok=True) + (case_dir / "prompt.md").write_text( + template_header(version) + "\nCOUNTRY: US\nClassify this miss.\n" + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts" / runner), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == f"scripts/{runner}" + assert meta["judge_template_version"] == version + assert template_version_problems(audit_dir) == [] + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_codex_publishes_no_verdict_for_a_prompt_rewritten_while_it_judged( + tmp_path: Path, +): + """Codex judges a v1 prompt; audit-prepare rewrites prompt.md to v2 while + it runs. The verdict describes bytes prompt.md no longer holds, so the + runner publishes neither it nor a sidecar, and leaves no copy behind.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + v1 = template_header(1) + "\nCOUNTRY: US\nClassify this miss.\n" + v2 = template_header(2) + "\nCOUNTRY: US\nClassify this miss.\n" + (case_dir / "prompt.md").write_text(v1) + rewrite = tmp_path / "v2.md" + rewrite.write_text(v2) + verdict = tmp_path / "canned_verdict.json" + copies = tmp_path / "copies" + copies.mkdir() + _fake_cli( + tmp_path / "bin" / "codex", + 'out=""; while [ $# -gt 0 ]; do' + ' if [ "$1" = -o ]; then out="$2"; shift; fi; shift; done\n' + "cat >/dev/null\n" + 'echo "model: gpt-5.6-sol"\n' + f'cp "{rewrite}" "{case_dir / "prompt.md"}"\n' + f'cat "{verdict}" > "$out"\n', + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "TMPDIR": str(copies)}, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + assert f"[FAIL] {case_dir.name}" in result.stdout + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + assert list(copies.iterdir()) == [] + assert (case_dir / "prompt.md").read_text() == v2 + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +@pytest.mark.parametrize("runner", ["run_audit_claude.sh", "run_audit_codex.sh"]) +@pytest.mark.parametrize("earlier", [None, "{"]) +def test_an_interrupted_publish_never_leaves_a_verdict_without_its_sidecar( + tmp_path: Path, runner, earlier +): + """A worker killed between publishing its two files leaves at most a + sidecar without a verdict, never a verdict that audit-prepare would read + as a legacy v1 verdict with no record of the bytes its judge read. Here + the first move succeeds and the worker is then terminated. A case that + held an invalid verdict (``earlier``) loses it when the run starts, so + no verdict sits beside the new sidecar either. The next run judges the + case again and publishes both.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + (case_dir / "prompt.md").write_text( + template_header(1) + "\nCOUNTRY: US\nClassify this miss.\n" + ) + if earlier is not None: + (case_dir / "verdict.json").write_text(earlier) + shim = tmp_path / "shim" + shim.mkdir() + _fake_cli(shim / "mv", '/bin/mv "$@" || exit $?\nkill -TERM "$PPID"\n') + interrupted = subprocess.run( + ["bash", str(ROOT / "scripts" / runner), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "PATH": f"{shim}:{env['PATH']}"}, + cwd=tmp_path, + ) + assert f"[ok] {case_dir.name}" not in interrupted.stdout, interrupted.stdout + assert not (case_dir / "verdict.json").exists() + assert template_version_problems(audit_dir) == [] + result = subprocess.run( + ["bash", str(ROOT / "scripts" / runner), str(audit_dir)], + capture_output=True, + text=True, + env=env, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + assert meta["judge_runner"] == f"scripts/{runner}" + assert meta["judge_template_version"] == 1 + assert ( + meta["prompt_sha256"] + == hashlib.sha256((case_dir / "prompt.md").read_bytes()).hexdigest() + ) + assert template_version_problems(audit_dir) == [] + assert prompt_hash_problems(audit_dir) == [] + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_a_terminated_codex_worker_removes_its_prompt_copy(tmp_path: Path): + """The worker judging a case is terminated while Codex runs: it removes + its private copy of prompt.md and publishes nothing.""" + audit_dir, case_dir, env = _claude_audit(tmp_path) + copies = tmp_path / "copies" + copies.mkdir() + _fake_cli( + tmp_path / "bin" / "codex", + 'cat >/dev/null\necho "model: gpt-5.6-sol"\nkill -TERM "$PPID"\n', + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "TMPDIR": str(copies)}, + cwd=tmp_path, + ) + assert "audit complete: 0/1" in result.stdout, result.stdout + result.stderr + assert list(copies.iterdir()) == [] + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + + +@pytest.mark.skipif(sys.platform == "win32", reason="bash runners") +def test_an_audit_prepare_between_a_publishs_two_moves_strands_nothing( + tmp_path: Path, +): + """End to end, the round-3 review's schedule on the real Codex runner: a + $250 case holding a truncated verdict is judged, and right after the + runner moves its sidecar into place an audit-prepare re-runs m1 at $999 + on v1; then the runner moves its verdict. The verdict keeps its sidecar, + whose prompt hash is the $250 prompt's: audit-collect refuses it and the + next audit-prepare re-opens it.""" + from policybench.audit import prepare_audit + + board = tmp_path / "us" + board.mkdir() + + def write_board(answer: float) -> None: + (board / "reference_outputs.csv").write_text( + "scenario_id,variable,value\ns0,snap,0.0\n" + ) + (board / "predictions.csv").write_text( + "model,scenario_id,variable,prediction,explanation,error\n" + f"m1,s0,snap,{answer},Estimated {answer}.,\n" + ) + + write_board(250.0) + audit_dir = tmp_path / "audit" + (case,) = prepare_audit(board, audit_dir, template_version=1) + case_dir = audit_dir / "cases" / case.case_id + judged = (case_dir / "prompt.md").read_bytes() + (case_dir / "verdict.json").write_text("{") + _, _, env = _claude_audit(tmp_path / "fixture") + hook = tmp_path / "prepare_999.py" + hook.write_text( + "from pathlib import Path\n" + "from policybench.audit import prepare_audit\n" + f"board = Path({str(board)!r})\n" + "(board / 'predictions.csv').write_text(\n" + " 'model,scenario_id,variable,prediction,explanation,error\\n'\n" + " 'm1,s0,snap,999.0,Estimated 999.0.,\\n'\n" + ")\n" + f"prepare_audit(board, Path({str(audit_dir)!r}), template_version=1)\n" + ) + shim = tmp_path / "shim" + shim.mkdir() + _fake_cli( + shim / "mv", + '/bin/mv "$@" || exit $?\n' + 'case "$3" in *verdict.meta.json)\n' + f' PYTHONPATH="{ROOT}" "{sys.executable}" "{hook}" || exit 1 ;;\n' + "esac\n", + ) + result = subprocess.run( + ["bash", str(ROOT / "scripts/run_audit_codex.sh"), str(audit_dir)], + capture_output=True, + text=True, + env={**env, "PATH": f"{shim}:{env['PATH']}"}, + cwd=tmp_path, + ) + assert result.returncode == 0, result.stderr + result.stdout + assert f"[ok] {case_dir.name}" in result.stdout, result.stdout + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + verdict = (case_dir / "verdict.json").read_bytes() + assert meta["verdict_sha256"] == hashlib.sha256(verdict).hexdigest() + assert meta["prompt_sha256"] == hashlib.sha256(judged).hexdigest() + assert b"999" in (case_dir / "prompt.md").read_bytes() + assert [c for c, _ in prompt_hash_problems(audit_dir)] == [case_dir.name] + prepare_audit(board, audit_dir, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() diff --git a/tests/test_judge_template.py b/tests/test_judge_template.py new file mode 100644 index 00000000..8b86f41e --- /dev/null +++ b/tests/test_judge_template.py @@ -0,0 +1,1270 @@ +"""The diagnosis judge's template versions, and how a tree of verdicts keeps +the version each was judged on. + +Invariants: +- every committed prompt re-renders on its recorded version: each prompt + sha256 that docs/gpt61sol/ and docs/haiku55/ commit re-renders, on the + version its verdict records, from the committed board of the release that + judged it (local only, for it needs the audit grounding); and prepare_audit + keeps a judged case's files only on the exact bytes its judge read: the + case rendered on its recorded version (absent: v1), equal to prompt.md and + to the sidecar's prompt_sha256 when it records one; +- v2 differs from v1 only by the dropped clause: the header diff is one + deletion, V1_REVIEW_CLAIM, and the rest of a prompt does not depend on the + version; +- version selection is explicit, never inferred: render_case_prompt, + prepare_audit and audit-prepare require a version, every call names one, + and prepare_audit keeps a verdict on its recorded version alone, never on + the version a prompt begins with; +- a version's header never changes (pinned by sha256), and no header is a + prefix of another, so a prompt's version is read off its bytes; +- after prepare_audit, a tree validates (template_version_problems is empty), + and a second prepare_audit changes nothing. +""" + +from __future__ import annotations + +import ast +import contextlib +import difflib +import functools +import hashlib +import inspect +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path +from unittest import mock + +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from policybench.audit import ( + AuditCase, + WrongModel, + build_audit_cases, + collect_audit, + prepare_audit, + prompt_hash_problems, + render_case_prompt, + template_version_problems, +) +from policybench.judge_template import ( + CURRENT_TEMPLATE_VERSION, + JUDGE_TEMPLATE_HEADERS, + TEMPLATE_VERSION_FIELD, + UNRECORDED_TEMPLATE_VERSION, + V1_REVIEW_CLAIM, + recorded_template_version, + template_header, + template_version_of, +) + +ROOT = Path(__file__).resolve().parents[1] +# Properties run on a loaded machine too: no deadline, no slow-input check. +PROPERTY = settings( + deadline=None, suppress_health_check=[HealthCheck.too_slow], max_examples=50 +) +# Each version's header, by sha256 of its UTF-8 bytes. A carried verdict was +# judged on exactly these bytes: never edit a pin, add a version. +PINNED_HEADER_SHA256 = { + 1: "25b91bdcea41c37d4aa7d105d55d4b68cf39f34b153cc0fc877a2848317572e2", + 2: "7d8ff9b27d4b0341310267c11ea895f84d403a3219b6eddb8d0bf7dc4b1354ed", +} +# origin/main before template versions: policybench/audit.py's _PROMPT_HEADER +# there is the template every verdict until then was judged on. +UNVERSIONED_COMMIT = "9ce4ade8382962a9134860c23f56d92509b5e57f" + + +# --- The versions -------------------------------------------------------------- + + +def test_every_version_is_pinned(): + assert set(JUDGE_TEMPLATE_HEADERS) == set(PINNED_HEADER_SHA256) + for version, header in JUDGE_TEMPLATE_HEADERS.items(): + digest = hashlib.sha256(header.encode("utf-8")).hexdigest() + assert digest == PINNED_HEADER_SHA256[version], version + + +def test_v1_is_the_template_before_versions(): + """Differential: v1 is the header origin/main rendered before versions.""" + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{UNVERSIONED_COMMIT}:policybench/audit.py"], + capture_output=True, + ) + if result.returncode: + pytest.fail( + f"cannot read policybench/audit.py at {UNVERSIONED_COMMIT[:12]}; fetch " + f"full history (git fetch --unshallow): {result.stderr.decode().strip()}" + ) + tree = ast.parse(result.stdout.decode("utf-8")) + (header,) = [ + node.value.value + for node in tree.body + if isinstance(node, ast.Assign) + and [getattr(t, "id", None) for t in node.targets] == ["_PROMPT_HEADER"] + ] + assert JUDGE_TEMPLATE_HEADERS[1] == header + + +def test_the_current_version_is_the_newest_and_an_unrecorded_one_is_v1(): + assert CURRENT_TEMPLATE_VERSION == max(JUDGE_TEMPLATE_HEADERS) == 2 + assert UNRECORDED_TEMPLATE_VERSION == 1 + + +def test_v2_is_v1_without_the_review_claim(): + v1, v2 = JUDGE_TEMPLATE_HEADERS[1], JUDGE_TEMPLATE_HEADERS[2] + assert v1.count(V1_REVIEW_CLAIM) == 1 + assert v2 == v1.replace(V1_REVIEW_CLAIM, "", 1) + for phrase in ( + "has survived an adversarial review program", + "the few real bugs found were fixed before this run", + ): + assert phrase in v1 and phrase not in v2 + assert ( + "generated directly from the engine's computation trace. Treat the " + "reference and its derivation as correct." in " ".join(v2.split()) + ) + + +def test_the_header_diff_is_one_deletion_of_the_clause(): + """v2 differs from v1 only by the dropped clause, and no diff can place + the deletion anywhere else: the common prefix ends where the clause + starts, the common suffix starts where it ends, and difflib's edit is that + one deletion.""" + v1, v2 = JUDGE_TEMPLATE_HEADERS[1], JUDGE_TEMPLATE_HEADERS[2] + prefix = len(os.path.commonprefix([v1, v2])) + suffix = len(os.path.commonprefix([v1[::-1], v2[::-1]])) + assert prefix + suffix == len(v2) + assert v1[prefix : len(v1) - suffix] == V1_REVIEW_CLAIM + opcodes = difflib.SequenceMatcher(None, v1, v2, autojunk=False).get_opcodes() + edits = [ + (tag, v1[i1:i2], v2[j1:j2]) for tag, i1, i2, j1, j2 in opcodes if tag != "equal" + ] + assert edits == [("delete", V1_REVIEW_CLAIM, "")] + + +def test_no_header_is_a_prefix_of_another(): + for version, header in JUDGE_TEMPLATE_HEADERS.items(): + for other, other_header in JUDGE_TEMPLATE_HEADERS.items(): + if version != other: + assert not other_header.startswith(header), (version, other) + + +def test_an_unknown_version_cannot_be_rendered(): + for version in (0, 3, True, "1", None, 1.0): + with pytest.raises(ValueError, match="no judge template version"): + template_header(version) + + +@PROPERTY +@given( + st.one_of( + st.none(), + st.booleans(), + st.integers(-5, 10), + st.floats(allow_nan=False), + st.text(max_size=3), + st.lists(st.integers(), max_size=2), + ) +) +def test_recorded_version_is_the_field_when_it_names_one(value): + expected = value if type(value) is int and value in JUDGE_TEMPLATE_HEADERS else None + assert recorded_template_version({TEMPLATE_VERSION_FIELD: value}) == expected + assert recorded_template_version({"judge_runner": "x"}) == 1 + # No sidecar records nothing. + assert recorded_template_version(None) is None + + +# --- Rendering ------------------------------------------------------------------- + +TEXT = st.text( + alphabet=st.characters(blacklist_categories=("Cs",)), + max_size=60, +) + + +@st.composite +def audit_cases(draw) -> AuditCase: + models = draw( + st.lists( + st.from_regex(r"[a-z][a-z0-9.-]{0,12}", fullmatch=True), + min_size=1, + max_size=4, + unique=True, + ) + ) + return AuditCase( + case_id=draw(st.from_regex(r"us__s[0-9]{1,3}__[a-z_]{1,12}", fullmatch=True)), + country=draw(st.sampled_from(["us", "uk"])), + scenario_id=draw(st.from_regex(r"s[0-9]{1,3}", fullmatch=True)), + variable=draw(st.from_regex(r"[a-z_]{1,20}", fullmatch=True)), + metric_type=draw(st.sampled_from(["amount", "binary"])), + reference_value=draw(st.sampled_from(["$0", "$1,234", "Yes", "No"])), + reference_derivation=draw(TEXT), + question=draw(TEXT), + wrong_models=tuple( + WrongModel( + model=model, + prediction=draw(st.sampled_from(["$250", "$0", "missing", "No"])), + explanation=draw(TEXT), + ) + for model in models + ), + grounding=draw(TEXT), + ) + + +@PROPERTY +@given(audit_cases()) +def test_a_prompts_version_is_read_off_its_bytes(case): + for version in JUDGE_TEMPLATE_HEADERS: + prompt = render_case_prompt(case, template_version=version) + assert prompt.startswith(template_header(version)) + assert template_version_of(prompt) == version + assert template_version_of(prompt.encode("utf-8")) == version + + +@PROPERTY +@given(audit_cases()) +def test_only_the_header_depends_on_the_version(case): + bodies = { + render_case_prompt(case, template_version=version)[len(header) :] + for version, header in JUDGE_TEMPLATE_HEADERS.items() + } + assert len(bodies) == 1 + v1 = render_case_prompt(case, template_version=1) + v2 = render_case_prompt(case, template_version=2) + assert v2 == v1.replace(V1_REVIEW_CLAIM, "", 1) + # The one deletion is the clause, where the header states it. + start = v1.index(V1_REVIEW_CLAIM) + assert v2 == v1[:start] + v1[start + len(V1_REVIEW_CLAIM) :] + assert start == JUDGE_TEMPLATE_HEADERS[1].index(V1_REVIEW_CLAIM) + + +@PROPERTY +@given(st.binary(max_size=80)) +def test_a_prompt_without_a_header_has_no_version(data): + if not any(data.startswith(h.encode()) for h in JUDGE_TEMPLATE_HEADERS.values()): + assert template_version_of(data) is None + assert template_version_of(b"\xff\xfe" + data) is None + + +# --- prepare_audit --------------------------------------------------------------- + + +def _board(directory: Path, answers: dict[str, float]) -> Path: + """A US run in which model m1 misses each scenario's $0 SNAP reference + with the given answer.""" + directory.mkdir(parents=True, exist_ok=True) + pd.DataFrame( + [{"scenario_id": s, "variable": "snap", "value": 0.0} for s in answers] + ).to_csv(directory / "reference_outputs.csv", index=False) + pd.DataFrame( + [ + { + "model": "m1", + "scenario_id": s, + "variable": "snap", + "prediction": answer, + "explanation": f"Estimated {answer}.", + "error": None, + } + for s, answer in answers.items() + ] + ).to_csv(directory / "predictions.csv", index=False) + return directory + + +VERDICT = { + "reference_suspect": False, + "reference_bug_hypothesis": "", + "case_failure_source": "llm_error", + "case_failure_subtype": "thresholds_rates", + "rationale": "Used the gross income test alone.", + "models": [ + { + "model": "m1", + "failure_source": "llm_error", + "failure_subtype": "thresholds_rates", + "diagnosis": "Applied the 2025 threshold.", + } + ], +} +# How a seed case's verdict records its template: the sidecar it gets. +SEED_STATES = { + "unjudged": None, + "no sidecar": None, + "v1, field absent": {}, + "v1": {TEMPLATE_VERSION_FIELD: 1}, + "v2": {TEMPLATE_VERSION_FIELD: 2}, + "unknown (null)": {TEMPLATE_VERSION_FIELD: None}, + "unknown (99)": {TEMPLATE_VERSION_FIELD: 99}, + "unknown (true)": {TEMPLATE_VERSION_FIELD: True}, +} +# The version a state's prompt was rendered with: an unjudged case's is the +# current one, as prepare_audit writes it; an unknown one's is v1's bytes. +SEED_PROMPT_VERSION = { + "unjudged": CURRENT_TEMPLATE_VERSION, + "no sidecar": 1, + "v1, field absent": 1, + "v1": 1, + "v2": 2, + "unknown (null)": 1, + "unknown (99)": 1, + "unknown (true)": 1, +} + + +def _judge(case_dir: Path, sidecar: dict | None) -> None: + """Write a verdict and, unless ``sidecar`` is None, its sidecar.""" + path = case_dir / "verdict.json" + path.write_text(json.dumps(VERDICT, indent=2, sort_keys=True)) + if sidecar is not None: + meta = { + "judge_runner": "scripts/run_audit_claude.sh", + "verdict_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + **sidecar, + } + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + + +def _seed(audit: Path, board: Path, states: dict[str, str]) -> dict[str, AuditCase]: + """Prepare ``board`` into ``audit``, then give each scenario's case the + prompt and verdict its state names. Returns the cases by scenario.""" + prepare_audit(board, audit, template_version=CURRENT_TEMPLATE_VERSION) + cases = {case.scenario_id: case for case in build_audit_cases(board)} + for scenario, state in states.items(): + case_dir = audit / "cases" / cases[scenario].case_id + (case_dir / "prompt.md").write_text( + render_case_prompt( + cases[scenario], template_version=SEED_PROMPT_VERSION[state] + ) + ) + if state != "unjudged": + _judge(case_dir, SEED_STATES[state]) + return cases + + +def _tree(audit: Path) -> dict[str, bytes]: + return { + str(path.relative_to(audit)): path.read_bytes() + for path in sorted(audit.rglob("*")) + if path.is_file() + } + + +@settings(max_examples=60, deadline=None, suppress_health_check=[HealthCheck.too_slow]) +@given( + st.dictionaries( + st.sampled_from(["s0", "s1", "s2", "s3"]), + st.tuples(st.sampled_from(sorted(SEED_STATES)), st.booleans()), + min_size=1, + ), + st.sampled_from(sorted(JUDGE_TEMPLATE_HEADERS)), +) +def test_prepare_keeps_each_judged_case_on_its_own_template(plan, version): + """Property over seed trees: a judged case whose case is unchanged keeps + its prompt, verdict and sidecar bytes when its recorded version is known; + every other case is (re-)opened on ``version``. The prepared tree + validates, and preparing again changes nothing.""" + with tempfile.TemporaryDirectory() as scratch: + root = Path(scratch) + board = _board(root / "us", {s: 250.0 for s in plan}) + audit = root / "audit" + states = {scenario: state for scenario, (state, _) in plan.items()} + before_cases = _seed(audit, board, states) + before = _tree(audit) + # Re-run m1 on the scenarios the plan changes: their prompts change. + changed = {s for s, (_, change) in plan.items() if change} + _board(board, {s: 999.0 if s in changed else 250.0 for s in plan}) + prepare_audit(board, audit, template_version=version) + cases = {case.scenario_id: case for case in build_audit_cases(board)} + after = _tree(audit) + for scenario, state in states.items(): + case_dir = f"cases/{cases[scenario].case_id}" + assert cases[scenario].case_id == before_cases[scenario].case_id + recorded = ( + recorded_template_version(SEED_STATES[state]) + if state != "unjudged" + else None + ) + carried = recorded is not None and scenario not in changed + if carried: + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + key = f"{case_dir}/{name}" + assert after.get(key) == before.get(key), (state, name) + assert template_version_of(after[f"{case_dir}/prompt.md"]) == recorded + else: + assert f"{case_dir}/verdict.json" not in after, state + assert f"{case_dir}/verdict.meta.json" not in after, state + assert after[f"{case_dir}/prompt.md"] == render_case_prompt( + cases[scenario], template_version=version + ).encode("utf-8") + assert template_version_problems(audit) == [] + assert prompt_hash_problems(audit) == [] + prepare_audit(board, audit, template_version=version) + assert _tree(audit) == after + + +def test_a_new_audit_on_the_current_version_drops_the_claim(tmp_path): + board = _board(tmp_path / "us", {"s0": 250.0, "s1": 100.0}) + audit = tmp_path / "audit" + cases = prepare_audit(board, audit, template_version=CURRENT_TEMPLATE_VERSION) + for case in cases: + prompt = (audit / "cases" / case.case_id / "prompt.md").read_bytes() + assert template_version_of(prompt) == CURRENT_TEMPLATE_VERSION == 2 + assert b"fixed before this run" not in prompt + assert b"survived an adversarial review program" not in prompt + + +def test_a_requested_version_renders_new_cases_and_an_unknown_one_is_refused( + tmp_path, +): + board = _board(tmp_path / "us", {"s0": 250.0}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + prompt = (audit / "cases" / case.case_id / "prompt.md").read_text() + assert prompt == render_case_prompt(case, template_version=1) + assert "fixed before this run" in prompt + with pytest.raises(ValueError, match="no judge template version 3"): + prepare_audit(board, tmp_path / "other", template_version=3) + assert not (tmp_path / "other").exists() + + +def test_a_v1_seed_carries_over_and_a_reopened_case_moves_to_v2(tmp_path): + """The fold drivers' flow, for a driver that names v2: a seed judged on v1 + (its sidecar predates the field) is copied into a stage; the case a new + model joins is re-opened on v2, and the other keeps the seed's bytes.""" + seed_board = _board(tmp_path / "seed" / "us", {"s0": 250.0, "s1": 100.0}) + seed = tmp_path / "seed" / "audit" + prepare_audit(seed_board, seed, template_version=1) + for case_dir in (seed / "cases").iterdir(): + _judge(case_dir, {}) + stage = tmp_path / "stage" / "audit" + stage.mkdir(parents=True) + for case_dir in (seed / "cases").iterdir(): + target = stage / "cases" / case_dir.name + target.mkdir(parents=True) + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + (target / name).write_bytes((case_dir / name).read_bytes()) + board = _board(tmp_path / "stage" / "us", {"s0": 250.0, "s1": 100.0}) + # A new model misses s0 and answers s1's $0 reference exactly. + predictions = pd.read_csv(board / "predictions.csv") + new = predictions.assign(model="new") + new.loc[new.scenario_id == "s1", "prediction"] = 0.0 + pd.concat([predictions, new]).to_csv(board / "predictions.csv", index=False) + cases = { + case.scenario_id: case + for case in prepare_audit(board, stage, template_version=2) + } + kept, reopened = (stage / "cases" / cases[s].case_id for s in ("s1", "s0")) + for name in ("prompt.md", "verdict.json", "verdict.meta.json"): + assert (kept / name).read_bytes() == ( + seed / "cases" / kept.name / name + ).read_bytes() + assert template_version_of((kept / "prompt.md").read_bytes()) == 1 + assert not (reopened / "verdict.json").exists() + assert (reopened / "prompt.md").read_text() == render_case_prompt( + cases["s0"], template_version=2 + ) + assert template_version_problems(stage) == [] + + +# --- A mixed tree ---------------------------------------------------------------- + + +def _mixed_tree(tmp_path: Path) -> tuple[Path, Path, dict[str, AuditCase]]: + board = _board(tmp_path / "us", {f"s{i}": 250.0 + i for i in range(5)}) + audit = tmp_path / "audit" + cases = _seed( + audit, + board, + { + "s0": "v1, field absent", + "s1": "v1, field absent", + "s2": "v1", + "s3": "v2", + "s4": "unjudged", + }, + ) + return board, audit, cases + + +def test_a_mixed_tree_validates_and_collects(tmp_path): + board, audit, cases = _mixed_tree(tmp_path) + assert template_version_problems(audit) == [] + out = collect_audit(board, audit) + assert out["template"].empty + assert list(out["template"].columns) == ["case_id", "problem"] + assert sorted(out["missing"]["case_id"]) == [cases["s4"].case_id] + assert len(out["case"]) == 4 + # Preparing it again, on either version, keeps every verdict: each judged + # case renders to its bytes on the version it records. Only the unjudged + # case follows the version the caller names. + before = _tree(audit) + unjudged = f"cases/{cases['s4'].case_id}/prompt.md" + for version in sorted(JUDGE_TEMPLATE_HEADERS): + prepare_audit(board, audit, template_version=version) + after = _tree(audit) + assert after[unjudged] == render_case_prompt( + cases["s4"], template_version=version + ).encode("utf-8") + assert {k: v for k, v in after.items() if k != unjudged} == { + k: v for k, v in before.items() if k != unjudged + } + assert _tree(audit) == before + _collect_cli(board, audit, tmp_path / "out") + assert (tmp_path / "out" / "us_audit_case_annotations.csv").is_file() + + +def test_a_verdict_whose_template_disagrees_with_its_prompt_is_flagged(tmp_path): + board, audit, cases = _mixed_tree(tmp_path) + case_dir = {s: audit / "cases" / case.case_id for s, case in cases.items()} + + def meta(scenario: str, **fields) -> None: + path = case_dir[scenario] / "verdict.meta.json" + path.write_text(json.dumps({**json.loads(path.read_text()), **fields})) + + meta("s1", judge_template_version=2) # records v2 on a v1 prompt + sidecar = case_dir["s3"] / "verdict.meta.json" + stripped = json.loads(sidecar.read_text()) + del stripped[TEMPLATE_VERSION_FIELD] + sidecar.write_text(json.dumps(stripped)) # v2 prompt, field absent (v1) + meta("s2", judge_template_version="2") # names no version + (case_dir["s0"] / "prompt.md").unlink() # a verdict without its prompt + _judge(case_dir["s4"], {}) # judged, sidecar without the field (v1) + (case_dir["s4"] / "prompt.md").write_text("Classify this miss.\n") + problems = dict(template_version_problems(audit)) + assert problems == { + cases["s0"].case_id: "a verdict without prompt.md", + cases["s1"].case_id: "prompt.md is v1, but its verdict records v2", + cases["s2"].case_id: ( + "its sidecar's judge_template_version '2' names no template " + "version ([1, 2])" + ), + cases["s3"].case_id: "prompt.md is v2, but its verdict records v1", + cases["s4"].case_id: "prompt.md is no template version, but its verdict " + "records v1", + } + out = collect_audit(board, audit) + assert dict(out["template"].itertuples(index=False)) == problems + with pytest.raises(SystemExit, match="5 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + assert not (tmp_path / "out").exists() + # Preparing again re-opens every flagged case. The verdict that lost its + # prompt has no prompt_sha256, so nothing records the bytes its judge + # read. + prepare_audit(board, audit, template_version=2) + assert template_version_problems(audit) == [] + for scenario in ("s0", "s1", "s2", "s3", "s4"): + assert not (case_dir[scenario] / "verdict.json").exists(), scenario + assert (case_dir[scenario] / "prompt.md").read_bytes() == render_case_prompt( + cases[scenario], template_version=2 + ).encode("utf-8") + + +def _judged_case(tmp_path: Path, answer: float = 250.0, **sidecar): + """A one-case board whose case is judged on v1, with ``sidecar``'s fields + in its sidecar. Returns the board, the audit, the case and its dir.""" + board = _board(tmp_path / "us", {"s0": answer}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + _judge(case_dir, {TEMPLATE_VERSION_FIELD: 1, **sidecar}) + return board, audit, case, case_dir + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def test_a_verdict_without_its_prompt_stands_only_on_its_recorded_hash(tmp_path): + """Without prompt.md, the sidecar's prompt_sha256 is the only record of + what the judge read. A verdict whose hash is the case rendered on its + version keeps its files, and its prompt comes back byte for byte; one + with no hash, or another hash, is re-opened.""" + for name, recorded in ( + ("matching", "v1"), + ("no hash", None), + ("another hash", "v2"), + ): + root = tmp_path / name.replace(" ", "-") + board = _board(root / "us", {"s0": 250.0}) + audit = root / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + renders = { + v: render_case_prompt(case, template_version=v).encode("utf-8") + for v in JUDGE_TEMPLATE_HEADERS + } + fields = {TEMPLATE_VERSION_FIELD: 1} + if recorded is not None: + fields["prompt_sha256"] = _sha256(renders[int(recorded[1])]) + _judge(case_dir, fields) + verdict = (case_dir / "verdict.json").read_bytes() + (case_dir / "prompt.md").unlink() + assert dict(template_version_problems(audit)) == { + case.case_id: "a verdict without prompt.md" + } + prepare_audit(board, audit, template_version=2) + if recorded == "v1": + assert (case_dir / "verdict.json").read_bytes() == verdict, name + assert (case_dir / "prompt.md").read_bytes() == renders[1], name + else: + assert not (case_dir / "verdict.json").exists(), name + assert not (case_dir / "verdict.meta.json").exists(), name + assert (case_dir / "prompt.md").read_bytes() == renders[2], name + assert template_version_problems(audit) == [] + + +def test_a_changed_case_without_its_prompt_is_reopened(tmp_path): + """The review's case: a verdict judged on m1's $250 answer loses its + prompt.md, and m1 is re-run and answers $999. The case renders anew, so + the verdict is dropped, even though its sidecar hashes the old prompt.""" + board, audit, case, case_dir = _judged_case(tmp_path) + old = (case_dir / "prompt.md").read_bytes() + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + meta["prompt_sha256"] = _sha256(old) + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + (case_dir / "prompt.md").unlink() + _board(board, {"s0": 999.0}) + (changed,) = prepare_audit(board, audit, template_version=1) + assert changed.case_id == case.case_id + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + new = (case_dir / "prompt.md").read_bytes() + assert new != old and b"999" in new + + +def test_a_sidecar_hash_that_is_not_the_prompt_reopens_the_case(tmp_path): + """A prompt.md that renders right but is not the bytes the sidecar says + its judge read is not the verdict's prompt.""" + board, audit, case, case_dir = _judged_case(tmp_path, prompt_sha256="0" * 64) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + + +def test_collect_refuses_a_verdict_published_on_rewritten_bytes(tmp_path): + """The review's race: a runner's judge reads the $250 case on v1; an + audit-prepare racing it re-runs m1 at $999 and rewrites prompt.md, on the + same version; then the runner publishes. The verdict's sidecar truthfully + hashes the $250 prompt, so audit-collect refuses it, and audit-prepare + re-opens it.""" + board = _board(tmp_path / "us", {"s0": 250.0}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + judged = (case_dir / "prompt.md").read_bytes() + _board(board, {"s0": 999.0}) + prepare_audit(board, audit, template_version=1) + rewritten = (case_dir / "prompt.md").read_bytes() + assert rewritten != judged + _judge(case_dir, {TEMPLATE_VERSION_FIELD: 1, "prompt_sha256": _sha256(judged)}) + assert template_version_problems(audit) == [] + problem = "prompt.md is not the bytes its judge read (its sidecar's prompt_sha256)" + assert prompt_hash_problems(audit) == [(case.case_id, problem)] + out = collect_audit(board, audit) + assert list(out["prompt_hash"].itertuples(index=False)) == [(case.case_id, problem)] + with pytest.raises(SystemExit, match="1 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + assert not (tmp_path / "out").exists() + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert (case_dir / "prompt.md").read_bytes() == rewritten + assert prompt_hash_problems(audit) == [] + + +def test_prepare_racing_a_publish_never_strands_the_verdict(tmp_path): + """The round-3 review's schedule, step by step. A $250 case holds a + truncated verdict.json. A runner's judge reads the $250 v1 prompt and the + runner publishes its sidecar. Before its verdict arrives, an + audit-prepare re-runs m1 at $999 on v1: it drops the truncated verdict + but leaves the sidecar, which describes another verdict. Then the + runner's verdict arrives. It still has its sidecar, whose prompt hash is + the $250 prompt's, so audit-collect refuses it and the next preparation + re-opens it.""" + board = _board(tmp_path / "us", {"s0": 250.0}) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + case_dir = audit / "cases" / case.case_id + judged = (case_dir / "prompt.md").read_bytes() + (case_dir / "verdict.json").write_text("{") + verdict = json.dumps(VERDICT, indent=2, sort_keys=True).encode() + sidecar = { + "judge_runner": "scripts/run_audit_codex.sh", + "verdict_sha256": _sha256(verdict), + "prompt_sha256": _sha256(judged), + TEMPLATE_VERSION_FIELD: 1, + } + (case_dir / "verdict.meta.json").write_text(json.dumps(sidecar)) + _board(board, {"s0": 999.0}) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert json.loads((case_dir / "verdict.meta.json").read_text()) == sidecar + (case_dir / "verdict.json").write_bytes(verdict) + assert [case_id for case_id, _ in prompt_hash_problems(audit)] == [case.case_id] + with pytest.raises(SystemExit, match="1 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert not (case_dir / "verdict.meta.json").exists() + + +def test_a_verdict_without_its_sidecar_is_refused_and_reopened(tmp_path): + """The round-4 review's schedule ends with a runner's verdict and no + sidecar. A $250 case's verdict is dropped by an audit-prepare re-running + m1 at $999, which pauses and then removes the sidecar the runner has just + published (the old sidecar matched the runner's verdict, whose bytes + repeat the old verdict's); then the runner's verdict arrives. Nothing + records what its judge read, so audit-collect refuses it and the next + audit-prepare re-opens it.""" + board, audit, case, case_dir = _judged_case(tmp_path) + verdict = (case_dir / "verdict.json").read_bytes() + _board(board, {"s0": 999.0}) + (case_dir / "verdict.json").unlink() + (case_dir / "verdict.meta.json").unlink() + (case_dir / "prompt.md").write_bytes( + render_case_prompt(build_audit_cases(board)[0], template_version=1).encode() + ) + (case_dir / "verdict.json").write_bytes(verdict) + problem = "a verdict without its sidecar (verdict.meta.json)" + assert template_version_problems(audit) == [(case.case_id, problem)] + with pytest.raises(SystemExit, match="1 verdicts disagree with their prompt.md"): + _collect_cli(board, audit, tmp_path / "out") + assert not (tmp_path / "out").exists() + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert template_version_problems(audit) == [] + + +def test_a_dropped_verdict_takes_only_its_own_sidecar(tmp_path): + """Re-opening a case removes its sidecar when the sidecar describes the + verdict (verdict_sha256) or records no verdict_sha256, as legacy sidecars + do; a sidecar describing another verdict stays for its runner.""" + for name, verdict_sha256, stays in ( + ("own", "verdict", False), + ("legacy", None, False), + ("another verdict's", "0" * 64, True), + ): + root = tmp_path / name.replace(" ", "-").replace("'", "") + board, audit, case, case_dir = _judged_case(root) + meta = json.loads((case_dir / "verdict.meta.json").read_text()) + if verdict_sha256 is None: + del meta["verdict_sha256"] + elif verdict_sha256 != "verdict": + meta["verdict_sha256"] = verdict_sha256 + (case_dir / "verdict.meta.json").write_text(json.dumps(meta)) + _board(board, {"s0": 999.0}) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists(), name + assert (case_dir / "verdict.meta.json").exists() == stays, name + + +def test_a_verdict_its_sidecar_does_not_describe_is_reopened(tmp_path): + """An unchanged case whose verdict.json is not the verdict its sidecar + describes is re-opened; the sidecar, another verdict's, stays.""" + board, audit, case, case_dir = _judged_case(tmp_path) + (case_dir / "verdict.json").write_text(json.dumps({**VERDICT, "rationale": "x"})) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert (case_dir / "verdict.meta.json").exists() + + +def test_prompts_compare_and_keep_exact_bytes(tmp_path): + """Byte equality, with no newline translation either way: a case whose + text holds a carriage return keeps its verdict and its prompt's bytes on + every preparation; a prompt.md with CRLF line endings is not the case's + rendering, so its verdict is re-opened and the prompt rewritten exactly.""" + board = _board(tmp_path / "us", {"s0": 250.0}) + predictions = pd.read_csv(board / "predictions.csv") + predictions["explanation"] = "Gross income\rtest only." + predictions.to_csv(board / "predictions.csv", index=False) + audit = tmp_path / "audit" + (case,) = prepare_audit(board, audit, template_version=1) + rendered = render_case_prompt(case, template_version=1).encode("utf-8") + assert b"Gross income\rtest only." in rendered + case_dir = audit / "cases" / case.case_id + assert (case_dir / "prompt.md").read_bytes() == rendered + _judge(case_dir, {TEMPLATE_VERSION_FIELD: 1, "prompt_sha256": _sha256(rendered)}) + before = _tree(audit) + for version in sorted(JUDGE_TEMPLATE_HEADERS): + prepare_audit(board, audit, template_version=version) + assert _tree(audit) == before, version + (case_dir / "prompt.md").write_bytes(rendered.replace(b"\n", b"\r\n")) + prepare_audit(board, audit, template_version=1) + assert not (case_dir / "verdict.json").exists() + assert (case_dir / "prompt.md").read_bytes() == rendered + + +def _collect_cli(board: Path, audit: Path, out: Path) -> None: + from policybench import cli + + argv = ["policybench", "audit-collect", "--country-dir", str(board)] + argv += ["--audit-dir", str(audit), "--output-dir", str(out)] + with mock.patch.object(sys, "argv", argv): + cli.main() + + +def test_audit_prepare_requires_a_template_version(tmp_path, capsys): + from policybench import cli + + board = _board(tmp_path / "us", {"s0": 250.0}) + for version in sorted(JUDGE_TEMPLATE_HEADERS): + audit = tmp_path / f"audit-{version}" + argv = ["policybench", "audit-prepare", "--country-dir", str(board)] + argv += ["--audit-dir", str(audit), "--template-version", str(version)] + with mock.patch.object(sys, "argv", argv): + cli.main() + (prompt,) = (audit / "cases").glob("*/prompt.md") + assert template_version_of(prompt.read_bytes()) == version + assert f"on judge template v{version}." in capsys.readouterr().out + # No flag: argparse refuses before anything is written. + argv = ["policybench", "audit-prepare", "--country-dir", str(board)] + argv += ["--audit-dir", str(tmp_path / "audit-none")] + with mock.patch.object(sys, "argv", argv): + with pytest.raises(SystemExit) as refused: + cli.main() + assert refused.value.code == 2 + assert "--template-version" in capsys.readouterr().err + assert not (tmp_path / "audit-none").exists() + argv = ["policybench", "audit-prepare", "--country-dir", str(board)] + argv += ["--audit-dir", str(tmp_path / "audit-9"), "--template-version", "9"] + with mock.patch.object(sys, "argv", argv): + with pytest.raises(SystemExit, match="no judge template version 9"): + cli.main() + assert not (tmp_path / "audit-9").exists() + + +# --- Version selection is explicit, never inferred ------------------------------ + + +def test_the_renderers_have_no_default_version(): + """Every renderer takes the version as a required keyword: none falls back + to CURRENT_TEMPLATE_VERSION, so adding a version moves no caller.""" + for function in (render_case_prompt, prepare_audit): + parameter = inspect.signature(function).parameters["template_version"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY, function.__name__ + assert parameter.default is inspect.Parameter.empty, function.__name__ + + +def test_a_renderer_called_without_a_version_is_refused(tmp_path): + board = _board(tmp_path / "us", {"s0": 250.0}) + (case,) = build_audit_cases(board) + with pytest.raises(TypeError, match="template_version"): + render_case_prompt(case) + with pytest.raises(TypeError): + render_case_prompt(case, CURRENT_TEMPLATE_VERSION) # not by name + with pytest.raises(TypeError, match="template_version"): + prepare_audit(board, tmp_path / "audit") + assert not (tmp_path / "audit").exists() + + +# The modules outside the tests that render judge prompts, and their calls. +RENDERER_CALLS = { + "policybench/audit.py": {"render_case_prompt": 2}, + "policybench/cli.py": {"prepare_audit": 1}, + "scripts/finish_adds0928.py": {"prepare_audit": 1}, + "scripts/finish_gpt61sol.py": {"prepare_audit": 1}, + "scripts/finish_haiku55.py": {"prepare_audit": 2, "render_case_prompt": 1}, +} +RENDERERS = ("render_case_prompt", "prepare_audit") + + +def _called_name(call: ast.Call) -> str | None: + func = call.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def test_every_renderer_call_names_its_version(): + """Static: every call of render_case_prompt or prepare_audit in + policybench/, scripts/ and reference_audit/ passes template_version by + name, and no value passed is read off a prompt (template_version_of).""" + found: dict[str, dict[str, int]] = {} + paths = [ + path + for part in ("policybench", "scripts", "reference_audit") + for path in sorted((ROOT / part).rglob("*.py")) + ] + for path in paths: + relative = path.relative_to(ROOT).as_posix() + for call in ast.walk(ast.parse(path.read_text(), filename=relative)): + if not isinstance(call, ast.Call) or _called_name(call) not in RENDERERS: + continue + name = _called_name(call) + counts = found.setdefault(relative, {}) + counts[name] = counts.get(name, 0) + 1 + keywords = {k.arg: k.value for k in call.keywords} + where = f"{relative}:{call.lineno} {name}" + assert "template_version" in keywords, f"{where} names no version" + inferred = [ + node + for node in ast.walk(keywords["template_version"]) + if isinstance(node, ast.Call) + and _called_name(node) == "template_version_of" + ] + assert not inferred, f"{where} reads its version off a prompt" + for relative, counts in RENDERER_CALLS.items(): + assert found.get(relative) == counts, relative + + +# Where Python code may read a version off a prompt: only to check a tree. +# (The runners' embedded Python also records what a judge read.) +TEMPLATE_VERSION_OF_CALLERS = {("policybench/audit.py", "template_version_problems")} + + +def test_only_the_tree_check_reads_a_version_off_a_prompt(): + """Static, closing the scan's gap for a version read into a variable and + passed on: template_version_of is called nowhere in policybench/, + scripts/ or reference_audit/ but in template_version_problems, which + renders nothing.""" + callers = set() + for part in ("policybench", "scripts", "reference_audit"): + for path in sorted((ROOT / part).rglob("*.py")): + relative = path.relative_to(ROOT).as_posix() + tree = ast.parse(path.read_text(), filename=relative) + for function in ast.walk(tree): + if not isinstance(function, ast.FunctionDef | ast.AsyncFunctionDef): + continue + for call in ast.walk(function): + if ( + isinstance(call, ast.Call) + and _called_name(call) == "template_version_of" + and function.name != "template_version_of" + ): + callers.add((relative, function.name)) + module_level = [ + node + for node in tree.body + if not isinstance( + node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef + ) + ] + for node in module_level: + for call in ast.walk(node): + if ( + isinstance(call, ast.Call) + and _called_name(call) == "template_version_of" + ): + callers.add((relative, "")) + assert callers == TEMPLATE_VERSION_OF_CALLERS + + +# The release drivers of releases judged before versions existed: each +# judged every verdict on v1. +V1_DRIVERS = ( + "scripts/finish_adds0928.py", + "scripts/finish_gpt61sol.py", + "scripts/finish_haiku55.py", +) + + +def test_each_past_release_driver_renders_on_v1_by_name(): + """Each driver states the version its release was judged on as the + module constant JUDGE_TEMPLATE_VERSION = 1, and passes that constant, not + a literal or another value, to every renderer it calls.""" + for relative in V1_DRIVERS: + tree = ast.parse((ROOT / relative).read_text(), filename=relative) + (value,) = [ + node.value + for node in tree.body + if isinstance(node, ast.Assign) + and [getattr(t, "id", None) for t in node.targets] + == ["JUDGE_TEMPLATE_VERSION"] + ] + assert isinstance(value, ast.Constant) and value.value == 1, relative + calls = [ + call + for call in ast.walk(tree) + if isinstance(call, ast.Call) and _called_name(call) in RENDERERS + ] + assert calls, relative + for call in calls: + (named,) = [k.value for k in call.keywords if k.arg == "template_version"] + assert isinstance(named, ast.Name), (relative, call.lineno) + assert named.id == "JUDGE_TEMPLATE_VERSION", (relative, call.lineno) + + +# A judged case's prompt.md, as the property draws it. +PROMPT_KINDS = ( + "v1", + "v2", + "v1 + text", + "v2 + text", + "v1 CRLF", + "v2 CRLF", + "no header", + "missing", +) +JUDGED_STATES = sorted(state for state in SEED_STATES if state != "unjudged") +# What the sidecar's prompt_sha256 hashes: nothing (no field), each version's +# rendering of the case, or bytes no prompt has. +HASHES = (None, "v1", "v2", "other") + + +@PROPERTY +@given( + st.sampled_from(JUDGED_STATES), + st.sampled_from(PROMPT_KINDS), + st.sampled_from(HASHES), + st.booleans(), + st.sampled_from(sorted(JUDGE_TEMPLATE_HEADERS)), + TEXT, +) +def test_a_verdict_is_kept_on_its_recorded_version_alone( + state, kind, hashed, described, version, text +): + """Never inferred, and byte for byte: prepare_audit keeps an unchanged + judged case's verdict exactly when the case rendered on the version its + sidecar records is the bytes its judge read. Those are prompt.md's bytes, + which must also hash to the sidecar's prompt_sha256 when it records one; + without prompt.md, that hash alone. A prompt.md on another version is + re-opened, though template_version_of reads a version off it and though + the caller may name that very version, and so is a CRLF copy. A sidecar + must also describe the verdict (its verdict_sha256); when ``described`` + is false, verdict.json is another verdict. A kept case's files are + untouched. A re-opened case loses its verdict, and its sidecar unless the + sidecar describes another verdict; it renders on the caller's version.""" + with tempfile.TemporaryDirectory() as scratch: + root = Path(scratch) + board = _board(root / "us", {"s0": 250.0}) + audit = root / "audit" + (case,) = prepare_audit(board, audit, template_version=version) + case_dir = audit / "cases" / case.case_id + renders = { + v: render_case_prompt(case, template_version=v).encode("utf-8") + for v in JUDGE_TEMPLATE_HEADERS + } + if kind == "missing": + prompt = None + elif kind == "no header": + prompt = f"Classify this miss.\n{text}".encode() + else: + prompt = renders[int(kind[1])] + if kind.endswith("+ text"): + prompt += f"\n{text}.".encode() + if kind.endswith("CRLF"): + prompt = prompt.replace(b"\n", b"\r\n") + if prompt is None: + (case_dir / "prompt.md").unlink() + else: + (case_dir / "prompt.md").write_bytes(prompt) + sidecar = SEED_STATES[state] + recorded_sha256 = None + if sidecar is not None and hashed is not None: + recorded_sha256 = ( + "0" * 64 if hashed == "other" else _sha256(renders[int(hashed[1])]) + ) + sidecar = {**sidecar, "prompt_sha256": recorded_sha256} + _judge(case_dir, sidecar) + if not described: + (case_dir / "verdict.json").write_text( + json.dumps({**VERDICT, "rationale": "Another verdict."}) + ) + other_verdict = sidecar is not None and not described + before = _tree(audit) + recorded = recorded_template_version(sidecar) + judged = renders.get(recorded) + kept = ( + not other_verdict + and judged is not None + and recorded_sha256 in (None, _sha256(judged)) + and (prompt == judged if prompt is not None else bool(recorded_sha256)) + ) + prepare_audit(board, audit, template_version=version) + after = _tree(audit) + where = (state, kind, hashed, described) + if kept: + restored = {f"cases/{case.case_id}/prompt.md": judged} + assert after == {**before, **restored}, where + else: + assert not (case_dir / "verdict.json").exists(), where + assert (case_dir / "verdict.meta.json").exists() == other_verdict, where + assert (case_dir / "prompt.md").read_bytes() == renders[version] + assert template_version_problems(audit) == [] + assert prompt_hash_problems(audit) == [] + + +# --- Every committed prompt re-renders on its recorded version ------------------ + +RUN_NAME = "us_full_run_20260612_policyengine_4_16_1_populace" +BOARD_PATH = Path("paper/snapshot/20260501/runs") / RUN_NAME +ANNOTATIONS_PATH = Path("annotations") / RUN_NAME +# The board files an audit renders from, and the annotations beside them. +BOARD_FILES = ( + "reference_outputs.csv", + "reference_outputs.csv.meta.json", + "reference_exclusions.json", + "scenarios.csv", + "scenarios.csv.meta.json", + "predictions.csv.gz", +) +ANNOTATION_FILES = ( + "us_audit_row_annotations.csv", + "us_case_notes.csv", + "us_case_reference_explanations.csv", + "us_adjudications.json", +) +# Each release PR's squash on main, which commits the board the release's +# audit renders from. A dashboard-data tag marks main when its GitHub release +# was cut, before that merge, so it does not hold the release's board. +RELEASE_COMMITS = { + "20260929": "d616e67c33b6f80dabf5cb7329f069f9a1de069d", + "20260930": "8b4c0ca146bb6f66deba6ce24009d49d70d92df2", + "20261010": "5a8164a001efb27fa55f47fe7ea26666a0de31f8", +} +# The audit grounding these releases rendered with, which the release drivers +# pin as GROUNDING_SHA256. It is not committed, so rendering is local only. +GROUNDING = Path( + "/Users/maxghenis/PolicyEngine/policybench/results/local/unified_audit/" + "grounding.csv" +) +GROUNDING_SHA256 = "b1e4a9bc74d762f410524a147efcda7d705c3afcfa3dc27f720fa60c54a7b55c" + + +def _seed_records(path: str) -> dict[str, dict]: + """A committed seed digest: each judged case's prompt and verdict sha256.""" + lines = (ROOT / path).read_text().splitlines() + assert lines[0] == "case_id,prompt_sha256,verdict_sha256" + return { + case: {"prompt_sha256": prompt, "verdict_sha256": verdict} + for case, prompt, verdict in (line.split(",") for line in lines[1:]) + } + + +def _provenance_records(path: str) -> dict[str, dict]: + """A committed judge provenance record: each re-judged verdict's entry.""" + verdicts = json.loads((ROOT / path).read_text())["verdicts"] + records = {entry["case_id"]: entry for entry in verdicts} + assert len(records) == len(verdicts), path + return records + + +def _committed_prompts() -> dict[str, dict[str, dict]]: + """Every committed prompt record, by the release whose audit holds it. + + Release 20260929's audit is docs/gpt61sol/seed_digest.csv. Release + 20260930 re-judged the cases docs/gpt61sol/judge_provenance.json lists + and carried the rest; release 20261006 carried that audit, as + docs/haiku55/seed_digest.csv records. Release 20261010 re-judged the + cases docs/haiku55/judge_provenance.json lists and carried the rest. + """ + seed_0929 = _seed_records("docs/gpt61sol/seed_digest.csv") + seed_1006 = _seed_records("docs/haiku55/seed_digest.csv") + return { + "20260929": seed_0929, + "20260930": { + **seed_0929, + **_provenance_records("docs/gpt61sol/judge_provenance.json"), + }, + "20261006": seed_1006, + "20261010": { + **seed_1006, + **_provenance_records("docs/haiku55/judge_provenance.json"), + }, + } + + +def test_the_committed_prompt_records_agree_and_name_a_version(): + """Runs anywhere: each committed record names a known template version + (none records the field, so each records v1), and release 20261006's + seed digest is release 20260930's audit: release 20260929's digest with + the prompts and verdicts of the cases release 20260930 re-judged.""" + committed = _committed_prompts() + assert {release: len(records) for release, records in committed.items()} == { + "20260929": 674, + "20260930": 674, + "20261006": 674, + # The seed's 674 and three cases release 20261010 added. + "20261010": 677, + } + for records in committed.values(): + for case, record in records.items(): + assert recorded_template_version(record) == 1, case + digests = { + release: { + case: (record["prompt_sha256"], record["verdict_sha256"]) + for case, record in records.items() + } + for release, records in committed.items() + } + assert digests["20261006"] == digests["20260930"] + + +@functools.cache +def _blob(commit: str, path: Path) -> bytes: + result = subprocess.run( + ["git", "-C", str(ROOT), "show", f"{commit}:{path.as_posix()}"], + capture_output=True, + ) + if result.returncode: + pytest.fail( + f"cannot read {path} at {commit[:12]}; fetch full history (git fetch " + f"--unshallow): {result.stderr.decode().strip()}" + ) + return result.stdout + + +@contextlib.contextmanager +def _object_strings(): + """Render as the release drivers did: with pandas' inferred Arrow strings + off.""" + if hasattr(pd.options, "future") and hasattr(pd.options.future, "infer_string"): + with pd.option_context("future.infer_string", False): + yield + else: + yield + + +@pytest.mark.slow +@pytest.mark.parametrize("release", sorted(RELEASE_COMMITS)) +def test_every_committed_prompt_rerenders_on_its_recorded_version(tmp_path, release): + """Local only: release ``release``'s board, read from the commit that + committed it, renders every judged case of its audit, on the version its + committed record names, to the prompt sha256 that record commits; and it + renders no other case. Release 20261006 judged nothing: it carried + release 20260930's audit, which test_the_committed_prompt_records_agree + checks, while rewording nine explanations that release 20261010 + re-opened, so its own board is not one an audit was rendered from.""" + if not GROUNDING.is_file(): + pytest.skip("the audit grounding is not on this machine") + assert hashlib.sha256(GROUNDING.read_bytes()).hexdigest() == GROUNDING_SHA256 + commit = RELEASE_COMMITS[release] + country_dir = tmp_path / RUN_NAME / "us" + country_dir.mkdir(parents=True) + for name in BOARD_FILES: + (country_dir / name).write_bytes(_blob(commit, BOARD_PATH / name)) + (tmp_path / RUN_NAME / "annotations").mkdir() + for name in ANNOTATION_FILES: + (tmp_path / RUN_NAME / "annotations" / name).write_bytes( + _blob(commit, ANNOTATIONS_PATH / name) + ) + grounding = pd.read_csv(GROUNDING) + lookup = { + (str(r.scenario_id), str(r.variable)): str(r.grounding) + for r in grounding.itertuples() + } + committed = _committed_prompts()[release] + rendered = {} + with _object_strings(): + for case in build_audit_cases(country_dir, grounding_lookup=lookup): + if case.to_manifest_row()["parse_failure_only"]: + continue + record = committed.get(case.case_id) + assert record is not None, f"{case.case_id} has no committed prompt" + prompt = render_case_prompt( + case, template_version=recorded_template_version(record) + ) + rendered[case.case_id] = hashlib.sha256(prompt.encode()).hexdigest() + expected = {case: record["prompt_sha256"] for case, record in committed.items()} + assert sorted(rendered) == sorted(expected) + differ = sorted(case for case in rendered if rendered[case] != expected[case]) + assert not differ, differ diff --git a/tests/test_run_audit_claude.py b/tests/test_run_audit_claude.py index c3c42839..f944801f 100644 --- a/tests/test_run_audit_claude.py +++ b/tests/test_run_audit_claude.py @@ -24,6 +24,7 @@ from hypothesis import strategies as st from policybench.audit import AUDIT_OUTPUT_SCHEMA +from policybench.judge_template import JUDGE_TEMPLATE_HEADERS, template_header ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "scripts")) @@ -370,6 +371,26 @@ def test_the_sidecar_binds_the_prompt_and_records_the_login(lane): assert f"Classify {name}." in transcript +@pytest.mark.parametrize("version", sorted(JUDGE_TEMPLATE_HEADERS)) +def test_the_sidecar_records_the_judge_template_version(lane, version): + """The sidecar records the template version the judged prompt begins with, + and null for a prompt audit-prepare did not render.""" + audit, _, _, run = lane + rendered, fixture = CASES + (audit / "cases" / rendered / "prompt.md").write_text( + template_header(version) + f"\nCOUNTRY: US\nClassify {rendered}.\n" + ) + result, _ = run() + assert result.returncode == 0, result.stderr + result.stdout + recorded = { + name: json.loads((audit / "cases" / name / "verdict.meta.json").read_text())[ + "judge_template_version" + ] + for name in CASES + } + assert recorded == {rendered: version, fixture: None} + + def test_a_home_lanes_own_account_is_recorded(lane): audit, _, _, run = lane home_login = {**DESKTOP, "email": "lane@x.org"} @@ -1239,6 +1260,7 @@ def _judge_both(work: Path, events: list, prompt: bytes, monkeypatch): argv += [str(config), "{}", "", _runner_value("DISALLOWED")] argv += [_runner_value("ATTACHMENTS"), "xhigh", str(work / "judged")] argv += [sha(prompt), _runner_value("PROMPT_NUDGE"), _runner_value("EVENT_TYPES")] + argv += [str(RUNNER.parent)] monkeypatch.setattr(sys, "argv", ["extract_verdict", *argv]) try: with contextlib.redirect_stderr(io.StringIO()):