From 864df114f10523eefa0ffbeb78e5c4d663f11952 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 14:36:48 -0400 Subject: [PATCH 1/9] Revive the UK transfer path on policyengine-uk 2.121.0 - Pin policyengine-uk 2.121.0 (core 3.32.17); a fresh 100-household US sample gives identical references under both cores (2,292 rows). - Re-pin the public transfer artifact to policyengine-uk-data 6b1f80e, which stores PIP component categories (the old pin's pip_*_reported columns were silently ignored after policyengine-uk#1656, zeroing PIP). - Prompt reference-period (uprated 2026-27) inputs instead of the stored 2025 values; income tax references matched the prompted facts in only 24 of 100 households before. - Compute UK references from the prompted facts, one PE-UK simulation per household (the 2026 UC health element is a seeded per-benefit-unit draw); keep the full-microdata calculation as a cross-check. - State derived facts the engine uses: sex and date of birth, education status at ages 16-19, LCWRA held before 6 April 2026. - Fail loudly when the artifact stores a variable the engine does not define or a prompted derived variable is renamed (is_child_or_QYP was removed). - Make the supervisor pass --country and use the country's output set. Co-Authored-By: Claude Opus 5.5 --- docs/benchmark_card.md | 28 ++++ policybench/ground_truth.py | 54 +++++++ policybench/policyengine_runtime.py | 16 +- policybench/prompts.py | 56 +++++++ policybench/scenarios.py | 157 +++++++++++++++++--- policybench/supervisor.py | 26 +++- pyproject.toml | 2 +- tests/test_eval_no_tools.py | 102 +++++++++++++ tests/test_ground_truth.py | 174 +++++++++++++++------- tests/test_policyengine_runtime.py | 2 +- tests/test_scenarios.py | 221 ++++++++++++++++++++++++++++ tests/test_supervisor.py | 58 +++++++- uv.lock | 8 +- 13 files changed, 809 insertions(+), 95 deletions(-) diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 145ee271..d0654cb5 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -409,6 +409,34 @@ as a public UK transfer path for benchmarking, not as a replacement for enhanced Family Resources Survey (FRS) microdata or as a population-representative UK household sample. +The UK path samples households from the public calibrated transfer dataset in +`PolicyEngine/policyengine-uk-data` (`enhanced_cps_2025.h5`, pinned to commit +`6b1f80e` and its sha256). Those records are PolicyEngine US households +re-weighted to UK targets, not UK survey records, and they carry no State +Pension: every pension-age person in the sample has none, and the prompt says +so by listing none. Since October 2026 the path works as follows. + +- **Reference-period inputs.** The artifact stores the 2025 survey year. + PolicyEngine UK uprates stored incomes, rents and savings when it calculates + 2026-27, so the prompt shows the uprated 2026-27 amounts the reference uses. +- **References from the prompted facts.** Each UK reference is a PolicyEngine + UK household simulation of exactly the facts the prompt lists, one simulation + per household, as on the US path. Facts the source record holds and the + prompt omits (for example Universal Credit deductions, or the seeded draw + that marks some 2026 health-element claimants as new) cannot move a + reference. `calculate_uk_transfer_microsimulation_values` recomputes the + outputs from the full records as a cross-check. +- **Derived facts stated.** Where PolicyEngine UK derives a fact that changes + an output, the prompt states it: each person's sex and date of birth (State + Pension age and the Pension Credit savings credit cutoff), the education + status of anyone aged 16 to 19 (qualifying young person rules), and an + adult's limited capability for work and work-related activity for Universal + Credit, held since before 6 April 2026. +- **Engine-version guards.** Reference generation stops if the artifact stores + a variable the installed PolicyEngine UK does not define, or if a derived + fact the prompt states is renamed, so a model release cannot silently drop + an input from the reference while it still reaches the prompt. + ## Naming discipline Public prose should prefer: diff --git a/policybench/ground_truth.py b/policybench/ground_truth.py index bbb1def7..a6ecb173 100644 --- a/policybench/ground_truth.py +++ b/policybench/ground_truth.py @@ -9,6 +9,7 @@ from policybench.config import DEFAULT_COUNTRY, TAX_YEAR, get_programs from policybench.policyengine_runtime import ( + get_uk_situation_simulation_class, get_us_situation_simulation_class, make_uk_transfer_microsimulation, ) @@ -480,6 +481,59 @@ def _calculate_ground_truth_uk( programs: list[str], year: int, ) -> pd.DataFrame: + """Calculate UK references from the facts each prompt states. + + The scenarios come from the transfer microdata, but the reference uses + only what the prompt lists, as on the US path. Values the source record + holds and the prompt omits cannot move a reference. + + Each scenario runs in its own simulation. PE-UK assigns the 2026 Universal + Credit health element by a seeded draw over the benefit units in a + simulation, so sharing one simulation would make a household's reference + depend on its position among the others. + """ + Simulation = get_uk_situation_simulation_class() + period = str(year) + rows = [] + for scenario in scenarios: + sim = Simulation(situation=scenario.to_pe_uk_situation()) + for variable in expand_programs_for_scenario(programs, scenario): + pe_variable = _pe_variable_for_output(variable, "uk") + entity_key = sim.tax_benefit_system.variables[pe_variable].entity.key + if entity_key not in {"person", "benunit", "household"}: + raise ValueError( + f"Unsupported UK entity '{entity_key}' for benchmark variable " + f"'{variable}'." + ) + rows.append( + { + "scenario_id": scenario.id, + "variable": variable, + "value": float( + np.asarray( + sim.calculate(pe_variable, period), dtype=float + ).sum() + ), + "impact_weight": None, + } + ) + return pd.DataFrame(rows) + + +def calculate_uk_transfer_microsimulation_values( + scenarios: list[Scenario], + programs: list[str] | None = None, + year: int = TAX_YEAR, +) -> pd.DataFrame: + """Calculate UK outputs from the full transfer microdata records. + + This was the scored UK reference before October 2026. It reads every + stored variable of each sampled household, including ones the prompt + omits, so it now serves as a cross-check: where it disagrees with the + scored reference, the source record holds a fact the prompt leaves out. + """ + if programs is None: + programs = get_programs("uk") os.environ.setdefault("HDF5_USE_FILE_LOCKING", "FALSE") sim = make_uk_transfer_microsimulation(get_uk_dataset_path()) period = str(year) diff --git a/policybench/policyengine_runtime.py b/policybench/policyengine_runtime.py index a60e43b0..bf9d1003 100644 --- a/policybench/policyengine_runtime.py +++ b/policybench/policyengine_runtime.py @@ -74,19 +74,19 @@ "runtime_dataset": "enhanced_cps_2025", "runtime_dataset_filename": "enhanced_cps_2025.h5", "runtime_dataset_repo": "PolicyEngine/policyengine-uk-data", - "runtime_dataset_pinned_commit": ("9514dfb7ec607897c9f7122a2e073b922c9fd8b6"), + "runtime_dataset_pinned_commit": ("6b1f80e0fdcd7ec149f347b4e0b2ae0081f0ff1b"), "runtime_dataset_pinned_url": ( "https://raw.githubusercontent.com/PolicyEngine/" - "policyengine-uk-data/9514dfb7ec607897c9f7122a2e073b922c9fd8b6/" + "policyengine-uk-data/6b1f80e0fdcd7ec149f347b4e0b2ae0081f0ff1b/" "policyengine_uk_data/storage/enhanced_cps_2025.h5" ), "runtime_dataset_uri": ( "policyengine_uk_data/storage/enhanced_cps_2025.h5 from the public " "PolicyEngine/policyengine-uk-data repository, pinned to commit " - "9514dfb7ec607897c9f7122a2e073b922c9fd8b6" + "6b1f80e0fdcd7ec149f347b4e0b2ae0081f0ff1b" ), "runtime_dataset_sha256": ( - "199ebc61d29231b4799ad337a95393765b5fb5aede1834b93ff2acecceded866" + "c663daea31b6fb8300f5c4758ccdcd7a227835ad7de90655294392fb95543eec" ), "runtime_dataset_note": ( "UK calibrated transfer dataset derived from benchmark-compatible " @@ -647,6 +647,14 @@ def get_uk_single_year_dataset_class(): return UKSingleYearDataset +def get_uk_situation_simulation_class(): + """Return the PE-UK situation Simulation class and record runtime metadata.""" + policyengine_release_bundle("uk") + from policyengine_uk import Simulation + + return Simulation + + def make_uk_transfer_microsimulation(dataset_path: str | Path): """Create a PE-UK Microsimulation for PolicyBench's public transfer data. diff --git a/policybench/prompts.py b/policybench/prompts.py index ba9354cf..6f719684 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -157,6 +157,9 @@ ), "veterans_benefits": "veterans benefits", "weekly_hours_worked": "hours worked per week", + "hours_worked": "annual hours worked", + "current_education": "current education", + "date_of_birth": "date of birth", "workers_compensation": "workers' compensation", "capital_gains_before_response": "capital gains", "communication_consumption": "communication spending", @@ -218,6 +221,51 @@ "weeks_unemployed", } +# PE-UK reads ``is_disabled_for_benefits`` as an adult's limited capability for +# work and work-related activity, and as a child's entitlement to the lower +# rate disabled child addition. PE-UK treats every such adult as having held +# the element before 6 April 2026 when a household is simulated on its own. +UK_DISABILITY_LABELS = { + "adult": ( + "has limited capability for work and work-related activity (LCWRA) " + "for Universal Credit, held since before 6 April 2026" + ), + "child": "is disabled (lower rate disabled child addition in Universal Credit)", +} + +MONTH_NAMES = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) + + +def _format_yyyymmdd(value) -> str: + number = int(value) + year, month, day = number // 10000, number // 100 % 100, number % 100 + return f"{day} {MONTH_NAMES[month - 1]} {year}" + + +# PE-UK enrolment stages, described in the terms the qualifying-young-person +# rules use (non-advanced versus advanced education). +UK_EDUCATION_DESCRIPTIONS = { + "NOT_IN_EDUCATION": "not in education or training", + "UPPER_SECONDARY": "secondary school", + "POST_SECONDARY": ( + "non-advanced education (school or college, below higher-education level)" + ), + "TERTIARY": "advanced (higher) education", +} + RATE_OR_RATIO_FIELD_SUFFIXES = ( "_rate", "_ratio", @@ -298,6 +346,10 @@ def _format_input_line(field: str, value, country: str = "us") -> str: if value: return f"- {label}" return f"- {label}: no" + if field == "date_of_birth": + return f"- {label}: {_format_yyyymmdd(value)}" + if field == "current_education" and value in UK_EDUCATION_DESCRIPTIONS: + return f"- {label}: {UK_EDUCATION_DESCRIPTIONS[value]}" if isinstance(value, str): return f"- {label}: {value.replace('_', ' ').title()}" if field == "selected_marketplace_plan_benchmark_ratio": @@ -331,6 +383,10 @@ def describe_person(person: Person, country: str = "us") -> str: for field, value in sorted(person.inputs.items()): if is_excluded_prompt_input_name(field): continue + if country == "uk" and field == "is_disabled_for_benefits" and value: + role = "adult" if person.name.startswith("adult") else "child" + lines.append(f"- {UK_DISABILITY_LABELS[role]}") + continue lines.append(_format_input_line(field, value, country=country)) return "\n".join(lines) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index a665c698..38590e17 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -331,6 +331,23 @@ "transport_consumption", } +# Person facts PE-UK derives rather than stores. ``state_pension`` is derived +# from the stored reported amount; ``current_education`` is imputed from age +# when the data carries no enrolment (16-17 non-advanced, 18-19 higher +# education), and decides qualifying-young-person status for Child Benefit and +# Universal Credit, so the prompt has to state it. ``date_of_birth`` places +# each person within their year of age (a seeded draw in microdata), which +# decides State Pension age and the Pension Credit savings credit cutoff. +UK_COMPUTED_PERSON_INPUTS = ( + "state_pension", + "current_education", + "date_of_birth", +) + +# Ages at which enrolment changes a qualifying-young-person or student rule. +# Below 16 every person is a child; from 20 no one is a qualifying young person. +UK_EDUCATION_PROMPT_AGES = range(16, 20) + UK_NON_PROMPTABLE_SENTINELS = { "", "NONE", @@ -342,10 +359,14 @@ ) UK_TRANSFER_DATASET_FILENAME = "enhanced_cps_2025.h5" +# Commit 6b1f80e (policyengine-uk-data#419, 2026-05-24) rebuilt the artifact +# with PIP component categories. The earlier pin (9514dfb, 2026-04-26) stored +# reported PIP amounts, which policyengine-uk stopped reading when +# policyengine-uk#1656 moved that mapping into the data package. UK_TRANSFER_DATASET_SHA256 = ( - "199ebc61d29231b4799ad337a95393765b5fb5aede1834b93ff2acecceded866" + "c663daea31b6fb8300f5c4758ccdcd7a227835ad7de90655294392fb95543eec" ) -UK_TRANSFER_DATASET_PINNED_COMMIT = "9514dfb7ec607897c9f7122a2e073b922c9fd8b6" +UK_TRANSFER_DATASET_PINNED_COMMIT = "6b1f80e0fdcd7ec149f347b4e0b2ae0081f0ff1b" UK_TRANSFER_DATASET_PINNED_URL = ( "https://raw.githubusercontent.com/PolicyEngine/policyengine-uk-data/" f"{UK_TRANSFER_DATASET_PINNED_COMMIT}/policyengine_uk_data/storage/" @@ -582,6 +603,43 @@ def to_pe_household(self) -> dict: "households": {"household": household_data}, } + def to_pe_uk_situation(self, prefix: str = "") -> dict: + """Convert to a PolicyEngine-UK situation holding only prompted facts. + + Each listed person, the region and the household inputs go in + unchanged; nothing else from the source record does, so the reference + is a function of the prompt. The benchmark keeps households with one + benefit unit, so every person joins that unit. ``prefix`` namespaces + entity keys when several scenarios share one simulation. + """ + if self.country != "uk": + raise ValueError("to_pe_uk_situation is only supported for UK scenarios.") + people = {} + for person in self.all_people: + person_data = { + "age": self._yearize(person.age), + # The stored wage leaf; employment_income adds the (zero) + # labour-supply response to it, as in the microsimulation. + "employment_income_before_lsr": self._yearize(person.employment_income), + } + for key, value in person.inputs.items(): + if key == "date_of_birth": + value = int(value) + person_data[key] = self._yearize(value) + people[f"{prefix}{person.name}"] = person_data + members = list(people) + household_data = { + "members": members, + "region": self._yearize(self.state), + } + for key, value in self.household_inputs.items(): + household_data[key] = self._yearize(value) + return { + "people": people, + "benunits": {f"{prefix}benunit": {"members": members}}, + "households": {f"{prefix}household": household_data}, + } + def marital_couple(self) -> tuple[str, str] | None: """The names of the tax unit's head and spouse, if the household has both. @@ -871,8 +929,44 @@ def get_uk_dataset_path() -> Path: ) -def load_uk_transfer_frames() -> tuple[pd.DataFrame, pd.DataFrame, int]: - """Load person and household frames from the local UK transfer artifact.""" +UK_REFERENCE_PERIOD_INPUT_ENTITIES = ("person", "household") + + +def _uk_dataset_inputs_by_entity( + values: dict[str, Any], tax_benefit_system: Any +) -> dict[str, list[str]]: + """Group the artifact's stored variables by PE-UK entity. + + Every stored variable must be one the installed policyengine-uk defines. + A variable the engine no longer reads (as ``pip_dl_reported`` became + after policyengine-uk#1656) would otherwise drop out of the reference + silently while the stale value still reached the prompt. + """ + unknown = sorted(key for key in values if key not in tax_benefit_system.variables) + if unknown: + raise ValueError( + "The UK transfer artifact stores variables the installed " + f"policyengine-uk does not define: {unknown}. Re-pin a transfer " + "artifact built for this model version." + ) + grouped: dict[str, list[str]] = {} + for key in values: + entity = tax_benefit_system.variables[key].entity.key + grouped.setdefault(entity, []).append(key) + return grouped + + +def load_uk_transfer_frames( + period: int = TAX_YEAR, +) -> tuple[pd.DataFrame, pd.DataFrame, int]: + """Load person and household frames from the local UK transfer artifact. + + The artifact stores one survey year (2025). PE-UK uprates stored incomes, + rents and savings when a later year is calculated, so each stored variable + is read back from the simulation at the reference ``period``. The prompt + then shows the same amounts the reference calculation uses, rather than + the stored base-year amounts. + """ from policybench.policyengine_runtime import ( get_uk_single_year_dataset_class, make_uk_transfer_microsimulation, @@ -884,18 +978,34 @@ def load_uk_transfer_frames() -> tuple[pd.DataFrame, pd.DataFrame, int]: dataset = UKSingleYearDataset(file_path=str(dataset_path)) values = dataset.load() - person_length = len(values["person_id"]) - household_length = len(values["household_id"]) + sim = make_uk_transfer_microsimulation(dataset_path) + period = str(period) + inputs_by_entity = _uk_dataset_inputs_by_entity(values, sim.tax_benefit_system) + + unsupported = sorted( + variable + for entity, variables in inputs_by_entity.items() + if entity not in UK_REFERENCE_PERIOD_INPUT_ENTITIES + for variable in variables + if not variable.endswith("_id") + ) + if unsupported: + raise ValueError( + "The UK transfer artifact stores benefit-unit inputs the prompt " + f"cannot show yet: {unsupported}." + ) - person_values = { - key: value for key, value in values.items() if len(value) == person_length - } - household_values = { - key: value for key, value in values.items() if len(value) == household_length - } + frames: dict[str, dict[str, np.ndarray]] = {} + for entity in UK_REFERENCE_PERIOD_INPUT_ENTITIES: + frames[entity] = { + variable: np.asarray( + sim.calculate(variable, period, map_to=entity, unweighted=True) + ) + for variable in inputs_by_entity.get(entity, []) + } - person_df = pd.DataFrame(person_values) - household_df = pd.DataFrame(household_values) + person_df = pd.DataFrame(frames["person"]) + household_df = pd.DataFrame(frames["household"]) person_df["person_id"] = pd.to_numeric( person_df["person_id"], errors="coerce" @@ -910,16 +1020,12 @@ def load_uk_transfer_frames() -> tuple[pd.DataFrame, pd.DataFrame, int]: pd.to_numeric(person_df["age"], errors="coerce").fillna(0).astype(int) ) - sim = make_uk_transfer_microsimulation(dataset_path) - period = str(TAX_YEAR) - for variable in ( - "state_pension", - "is_child_or_QYP", - "pip_dl_category", - "pip_m_category", - ): + for variable in UK_COMPUTED_PERSON_INPUTS: if variable not in sim.tax_benefit_system.variables: - continue + raise ValueError( + f"policyengine-uk no longer defines '{variable}', which the UK " + "transfer path prompts. Update UK_COMPUTED_PERSON_INPUTS." + ) calculated = np.asarray( sim.calculate( variable, @@ -1352,11 +1458,14 @@ def _extract_uk_person_inputs(row: pd.Series) -> dict[str, Any]: in { "age", *UK_EMPLOYMENT_INCOME_COLUMNS, - "gender", "marital_status", } ): continue + if col == "current_education" and int(row["age"]) not in ( + UK_EDUCATION_PROMPT_AGES + ): + continue promptable = _uk_promptable_value(value) if promptable is not None: inputs[col] = promptable diff --git a/policybench/supervisor.py b/policybench/supervisor.py index 095d175b..51358d23 100644 --- a/policybench/supervisor.py +++ b/policybench/supervisor.py @@ -51,7 +51,7 @@ import pandas as pd -from policybench.config import MODELS, PROGRAMS +from policybench.config import MODELS, get_programs from policybench.model_cards import ( PROMPT_CONTRACT_VERSION, answer_contract_for, @@ -187,11 +187,15 @@ def _load_scenarios(self): raise ValueError("Scenario manifest must contain at least one scenario.") return scenarios + def _programs_for(self, index: int) -> list[str]: + """The output set the worker CLI requests for one scenario's country.""" + return get_programs(self.scenarios[index].country) + def _load_initial_request_variables(self) -> list[str]: """Expand the first manifest row exactly as the worker CLI will.""" from policybench.spec import expand_programs_for_scenario - return expand_programs_for_scenario(PROGRAMS, self.scenarios[0]) + return expand_programs_for_scenario(self._programs_for(0), self.scenarios[0]) @staticmethod def _joined_sha256(values: list[str]) -> str: @@ -201,7 +205,15 @@ def _workload(self) -> dict: return { "manifest_sha256": hashlib.sha256(self.manifest.read_bytes()).hexdigest(), "scenario_ids_sha256": self._joined_sha256(self.scenario_ids), - "output_set_sha256": self._joined_sha256(sorted(PROGRAMS)), + "output_set_sha256": self._joined_sha256( + sorted( + { + program + for scenario in self.scenarios + for program in get_programs(scenario.country) + } + ) + ), "prompt_contract_version": PROMPT_CONTRACT_VERSION, } @@ -211,7 +223,9 @@ def scenario_csv(self, index: int) -> Path: def _expected_outputs_for_scenario(self, index: int) -> list[str]: from policybench.spec import expand_programs_for_scenario - return expand_programs_for_scenario(PROGRAMS, self.scenarios[index]) + return expand_programs_for_scenario( + self._programs_for(index), self.scenarios[index] + ) def _scenario_complete(self, index: int) -> bool: from policybench.eval_no_tools import NO_TOOLS_RESULT_COLUMNS @@ -310,7 +324,7 @@ def _expected_scenario_metadata(self, index: int) -> dict: task="eval_no_tools_batch", scenarios=[self.scenarios[index]], models={self.model: self.litellm_id}, - programs=PROGRAMS, + programs=self._programs_for(index), run_id=None, include_explanations=True, env=self.env, @@ -660,6 +674,8 @@ def _spawn(self, index: int) -> subprocess.Popen: "-m", "policybench.cli", "eval-no-tools", + "--country", + self.scenarios[index].country, "--model", self.model, "--scenario-manifest", diff --git a/pyproject.toml b/pyproject.toml index 8c564897..eadfb41a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "numpy>=1.24", "policyengine==6.1.2", "policyengine-us==2.38.6", - "policyengine-uk==2.89.0", + "policyengine-uk==2.125.1", "anthropic>=0.115.1", "google-genai>=2.10.0", ] diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 63812f31..c0b39159 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -3962,3 +3962,105 @@ def test_chunk_override_env_runs_whole_scenario(monkeypatch): monkeypatch.setenv("POLICYBENCH_CHUNK_OVERRIDE", "none") assert _required_explanation_chunk_size("claude-fable-5", True) is None assert _required_explanation_chunk_size("claude-sonnet-5", True) is None + + +def test_uk_prompt_states_role_specific_disability_and_birth_facts(): + scenario = Scenario( + id="uk-disability", + country="uk", + state="WALES", + filing_status=None, + adults=[ + Person( + name="adult1", + age=73, + employment_income=0.0, + inputs={ + "is_disabled_for_benefits": True, + "date_of_birth": 19530403.0, + "gender": "FEMALE", + "hours_worked": 1_820.0, + }, + ) + ], + children=[ + Person( + name="child1", + age=17, + employment_income=0.0, + inputs={ + "is_disabled_for_benefits": True, + "current_education": "POST_SECONDARY", + }, + ), + Person( + name="child2", + age=19, + employment_income=0.0, + inputs={"current_education": "TERTIARY"}, + ), + ], + year=2026, + ) + + prompt = make_no_tools_batch_prompt(scenario, ["universal_credit"]) + + assert ( + "- has limited capability for work and work-related activity (LCWRA) " + "for Universal Credit, held since before 6 April 2026" + ) in prompt + assert ( + "- is disabled (lower rate disabled child addition in Universal Credit)" + ) in prompt + assert "is disabled for benefits" not in prompt + assert "- date of birth: 3 April 1953" in prompt + assert "- gender: Female" in prompt + assert "- annual hours worked: 1,820" in prompt + assert ( + "- current education: non-advanced education (school or college, below " + "higher-education level)" + ) in prompt + assert "- current education: advanced (higher) education" in prompt + + +@pytest.mark.slow +def test_pe_uk_gives_a_lone_lcwra_household_the_protected_health_element(): + """The UK prompt says LCWRA was held before 6 April 2026; check PE-UK agrees. + + PE-UK draws new-claimant status per benefit unit from a fixed seed. A + household simulated alone takes the first draw, which is an existing + claimant, so it gets more than the new-claimant rate. If PE-UK changes the + draw, this fails and the prompt wording has to change with it. + """ + from policyengine_uk import CountryTaxBenefitSystem + + from policybench.ground_truth import calculate_ground_truth + + scenario = Scenario( + id="uk-lcwra", + country="uk", + state="NORTH_EAST", + filing_status=None, + adults=[ + Person( + name="adult1", + age=35, + employment_income=0.0, + inputs={"is_disabled_for_benefits": True}, + ) + ], + household_inputs={"tenure_type": "OWNED_OUTRIGHT"}, + year=2026, + ) + from policyengine_uk import Simulation + + sim = Simulation(situation=scenario.to_pe_uk_situation()) + lcwra = float(sim.calculate("uc_LCWRA_element", 2026)[0]) + new_claimant_monthly = float( + CountryTaxBenefitSystem().parameters.gov.dwp.universal_credit.rebalancing.new_claimant_health_element( + "2026-06-01" + ) + ) + assert lcwra > new_claimant_monthly * 12 + reference = calculate_ground_truth([scenario], ["universal_credit"], year=2026) + assert reference["value"].iloc[0] > 0 diff --git a/tests/test_ground_truth.py b/tests/test_ground_truth.py index 4b6a8540..f956b409 100644 --- a/tests/test_ground_truth.py +++ b/tests/test_ground_truth.py @@ -282,61 +282,80 @@ def test_non_boolean_variables_still_sum(self): ) -def test_calculate_ground_truth_uk_aggregates_native_entities(monkeypatch): - class FakeVariable: - def __init__(self, entity_key: str): - self.entity = SimpleNamespace(key=entity_key) - - class FakeSimulation: - def __init__(self, dataset): - self.tax_benefit_system = SimpleNamespace( - variables={ - "income_tax": FakeVariable("person"), - "child_benefit": FakeVariable("benunit"), - "housing_costs": FakeVariable("household"), - } +def _uk_scenario(scenario_id: str, *, rent: float, child_age: int | None = None): + children = ( + [Person(name="child1", age=child_age, employment_income=0.0)] + if child_age is not None + else [] + ) + return Scenario( + id=scenario_id, + country="uk", + state="WEST_MIDLANDS", + filing_status=None, + adults=[ + Person( + name="adult1", + age=40, + employment_income=18_000.0, + inputs={"gender": "FEMALE", "date_of_birth": 19860601.0}, ) + ], + children=children, + household_inputs={"rent": rent, "tenure_type": "RENT_PRIVATELY"}, + metadata={"household_id": 1}, + ) + - def calculate(self, variable, year, map_to, unweighted=True): - lookup = { - ("person_household_id", "person"): np.array([101, 101, 202]), - ("person_benunit_id", "person"): np.array([101, 101, 202]), - ("benunit_id", "benunit"): np.array([101, 202]), - ("household_id", "household"): np.array([101, 202]), - ("income_tax", "person"): np.array([100.0, 25.0, 80.0]), - ("child_benefit", "benunit"): np.array([40.0, 10.0]), - ("housing_costs", "household"): np.array([700.0, 300.0]), +class _FakeUKSimulation: + """Records each situation and returns per-entity values derived from it.""" + + situations: list = [] + + def __init__(self, situation): + type(self).situations.append(situation) + self.situation = situation + self.tax_benefit_system = SimpleNamespace( + variables={ + "income_tax": SimpleNamespace(entity=SimpleNamespace(key="person")), + "child_benefit": SimpleNamespace(entity=SimpleNamespace(key="benunit")), + "housing_costs": SimpleNamespace( + entity=SimpleNamespace(key="household") + ), } - return lookup[(variable, map_to)] + ) + def calculate(self, variable, period): + assert period == "2026" + people = self.situation["people"].values() + household = next(iter(self.situation["households"].values())) + if variable == "income_tax": + return np.array( + [ + person["employment_income_before_lsr"]["2026"] * 0.1 + for person in people + ] + ) + if variable == "child_benefit": + children = sum(1 for person in people if person["age"]["2026"] < 16) + return np.array([children * 100.0]) + if variable == "housing_costs": + return np.array([household["rent"]["2026"]]) + raise KeyError(variable) + + +def test_calculate_ground_truth_uk_runs_each_scenario_on_its_prompted_facts( + monkeypatch, +): + _FakeUKSimulation.situations = [] monkeypatch.setattr( ground_truth, - "make_uk_transfer_microsimulation", - lambda dataset_path: FakeSimulation(dataset_path), - ) - monkeypatch.setattr( - ground_truth, - "get_uk_dataset_path", - lambda: "/tmp/fake_enhanced_cps_uk.h5", + "get_uk_situation_simulation_class", + lambda: _FakeUKSimulation, ) - scenarios = [ - Scenario( - id="uk_1", - country="uk", - state="WEST_MIDLANDS", - filing_status=None, - adults=[Person(name="adult1", age=40, employment_income=0.0)], - metadata={"household_id": 101}, - ), - Scenario( - id="uk_2", - country="uk", - state="LONDON", - filing_status=None, - adults=[Person(name="adult1", age=35, employment_income=0.0)], - metadata={"household_id": 202}, - ), + _uk_scenario("uk_1", rent=7_000.0, child_age=4), + _uk_scenario("uk_2", rent=3_000.0), ] result = calculate_ground_truth( @@ -347,16 +366,61 @@ def calculate(self, variable, year, map_to, unweighted=True): expected = pd.DataFrame( [ - {"scenario_id": "uk_1", "variable": "child_benefit", "value": 40.0}, - {"scenario_id": "uk_1", "variable": "housing_costs", "value": 700.0}, - {"scenario_id": "uk_1", "variable": "income_tax", "value": 125.0}, - {"scenario_id": "uk_2", "variable": "child_benefit", "value": 10.0}, - {"scenario_id": "uk_2", "variable": "housing_costs", "value": 300.0}, - {"scenario_id": "uk_2", "variable": "income_tax", "value": 80.0}, + {"scenario_id": "uk_1", "variable": "child_benefit", "value": 100.0}, + {"scenario_id": "uk_1", "variable": "housing_costs", "value": 7_000.0}, + {"scenario_id": "uk_1", "variable": "income_tax", "value": 1_800.0}, + {"scenario_id": "uk_2", "variable": "child_benefit", "value": 0.0}, + {"scenario_id": "uk_2", "variable": "housing_costs", "value": 3_000.0}, + {"scenario_id": "uk_2", "variable": "income_tax", "value": 1_800.0}, ] - ).sort_values(["scenario_id", "variable"]) - + ) pd.testing.assert_frame_equal( result.drop(columns="impact_weight").reset_index(drop=True), expected.reset_index(drop=True), ) + # One simulation per scenario, so no household shares a simulation (and + # PE-UK's per-benefit-unit seeded draws) with another. + assert len(_FakeUKSimulation.situations) == 2 + assert all( + len(situation["households"]) == 1 and len(situation["benunits"]) == 1 + for situation in _FakeUKSimulation.situations + ) + + +def test_calculate_ground_truth_uk_does_not_depend_on_batch_composition(monkeypatch): + monkeypatch.setattr( + ground_truth, + "get_uk_situation_simulation_class", + lambda: _FakeUKSimulation, + ) + programs = ["income_tax", "child_benefit", "housing_costs"] + first = _uk_scenario("uk_1", rent=7_000.0, child_age=4) + second = _uk_scenario("uk_2", rent=3_000.0) + + together = calculate_ground_truth([second, first], programs=programs, year=2026) + alone = calculate_ground_truth([first], programs=programs, year=2026) + + pd.testing.assert_frame_equal( + together[together["scenario_id"] == "uk_1"].reset_index(drop=True), + alone.reset_index(drop=True), + ) + + +def test_calculate_ground_truth_uk_rejects_unsupported_entities(monkeypatch): + class UnsupportedEntitySimulation(_FakeUKSimulation): + def __init__(self, situation): + super().__init__(situation) + self.tax_benefit_system.variables["income_tax"] = SimpleNamespace( + entity=SimpleNamespace(key="state") + ) + + monkeypatch.setattr( + ground_truth, + "get_uk_situation_simulation_class", + lambda: UnsupportedEntitySimulation, + ) + + with pytest.raises(ValueError, match="Unsupported UK entity"): + calculate_ground_truth( + [_uk_scenario("uk_1", rent=1.0)], programs=["income_tax"], year=2026 + ) diff --git a/tests/test_policyengine_runtime.py b/tests/test_policyengine_runtime.py index 2adefb53..033b2402 100644 --- a/tests/test_policyengine_runtime.py +++ b/tests/test_policyengine_runtime.py @@ -32,7 +32,7 @@ def test_uk_policyengine_bundle_uses_transfer_artifact(monkeypatch): assert "private" not in bundle["default_dataset_uri"] assert bundle["runtime_dataset"] == "enhanced_cps_2025" assert bundle["runtime_dataset_sha256"] == ( - "199ebc61d29231b4799ad337a95393765b5fb5aede1834b93ff2acecceded866" + "c663daea31b6fb8300f5c4758ccdcd7a227835ad7de90655294392fb95543eec" ) assert "not native UK survey microdata" in bundle["runtime_dataset_note"] diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index c163280a..f2611e61 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -2,8 +2,11 @@ import json +import numpy as np import pandas as pd import pytest +from hypothesis import given, settings +from hypothesis import strategies as st import policybench.scenarios as scenarios_module from policybench.scenarios import ( @@ -1256,3 +1259,221 @@ def test_unstated_or_zero_hours_keep_the_engine_default(hours): head = _hours_scenario(hours).to_pe_household()["people"]["head"] assert "hours_worked_last_week" not in head assert "weekly_hours_worked_before_lsr" not in head + + +class _FakeUKVariable: + def __init__(self, entity_key: str): + self.entity = type("Entity", (), {"key": entity_key})() + + +class _FakeUKTransferSimulation: + """Stored values are the 2025 survey year; 2026 values are uprated.""" + + STORED = { + "person_id": ("person", [1.0, 2.0]), + "person_household_id": ("person", [10.0, 10.0]), + "person_benunit_id": ("person", [20.0, 20.0]), + "age": ("person", [40.0, 17.0]), + "gender": ("person", ["FEMALE", "MALE"]), + "employment_income_before_lsr": ("person", [30_000.0, 0.0]), + "benunit_id": ("benunit", [20.0]), + "household_id": ("household", [10.0]), + "household_weight": ("household", [1.0]), + "region": ("household", ["NORTH_WEST"]), + "rent": ("household", [6_000.0]), + } + UPRATED = {"employment_income_before_lsr": 1.034, "rent": 1.02} + COMPUTED = { + "state_pension": [0.0, 0.0], + "current_education": ["NOT_IN_EDUCATION", "POST_SECONDARY"], + "date_of_birth": [19860601, 20090401], + } + + def __init__(self, extra_stored=None, drop_computed=()): + stored = dict(self.STORED, **(extra_stored or {})) + self.stored = stored + variables = { + name: _FakeUKVariable(entity) for name, (entity, _) in stored.items() + } + for name in self.COMPUTED: + if name not in drop_computed: + variables[name] = _FakeUKVariable("person") + for name in extra_stored or {}: + if name.startswith("unknown_"): + variables.pop(name) + self.tax_benefit_system = type("TBS", (), {"variables": variables})() + self.periods = set() + + def calculate(self, variable, period, map_to=None, unweighted=True): + self.periods.add(period) + if variable in self.COMPUTED: + return np.asarray(self.COMPUTED[variable]) + values = np.asarray(self.stored[variable][1]) + if period == "2026" and variable in self.UPRATED: + return values * self.UPRATED[variable] + return values + + +def _patch_uk_transfer(monkeypatch, sim): + import policybench.policyengine_runtime as runtime + + class FakeDataset: + time_period = 2025 + + def __init__(self, file_path): + self.file_path = file_path + + def load(self): + return { + name: np.asarray(values) for name, (_, values) in sim.stored.items() + } + + monkeypatch.setattr(scenarios_module, "get_uk_dataset_path", lambda: "fake.h5") + monkeypatch.setattr( + runtime, "get_uk_single_year_dataset_class", lambda: FakeDataset + ) + monkeypatch.setattr(runtime, "make_uk_transfer_microsimulation", lambda path: sim) + + +def test_load_uk_transfer_frames_prompts_reference_period_values(monkeypatch): + sim = _FakeUKTransferSimulation() + _patch_uk_transfer(monkeypatch, sim) + + person_df, household_df, dataset_year = scenarios_module.load_uk_transfer_frames() + + assert dataset_year == 2025 + assert sim.periods == {"2026"} + # The prompt must show the uprated 2026-27 amounts the reference uses, + # not the stored 2025 survey values. + assert person_df["employment_income_before_lsr"].tolist() == pytest.approx( + [31_020.0, 0.0] + ) + assert household_df["rent"].tolist() == pytest.approx([6_120.0]) + assert person_df["current_education"].tolist() == [ + "NOT_IN_EDUCATION", + "POST_SECONDARY", + ] + assert person_df["date_of_birth"].tolist() == [19860601, 20090401] + + +def test_load_uk_transfer_frames_rejects_variables_the_engine_dropped(monkeypatch): + sim = _FakeUKTransferSimulation( + extra_stored={"unknown_pip_dl_reported": ("person", [0.0, 0.0])} + ) + _patch_uk_transfer(monkeypatch, sim) + + with pytest.raises(ValueError, match="does not define"): + scenarios_module.load_uk_transfer_frames() + + +def test_load_uk_transfer_frames_rejects_renamed_computed_inputs(monkeypatch): + sim = _FakeUKTransferSimulation(drop_computed=("current_education",)) + _patch_uk_transfer(monkeypatch, sim) + + with pytest.raises(ValueError, match="no longer defines 'current_education'"): + scenarios_module.load_uk_transfer_frames() + + +def test_load_uk_transfer_frames_rejects_benefit_unit_inputs(monkeypatch): + sim = _FakeUKTransferSimulation(extra_stored={"benunit_rent": ("benunit", [100.0])}) + _patch_uk_transfer(monkeypatch, sim) + + with pytest.raises(ValueError, match="benefit-unit inputs"): + scenarios_module.load_uk_transfer_frames() + + +def test_uk_current_education_is_prompted_only_for_ages_16_to_19(): + def person_row(age): + return pd.Series( + { + "person_id": 1, + "person_household_id": 1, + "person_benunit_id": 1, + "age": age, + "employment_income_before_lsr": 0.0, + "current_education": "POST_SECONDARY", + } + ) + + prompted = { + age: "current_education" + in scenarios_module._extract_uk_person_inputs(person_row(age)) + for age in (15, 16, 19, 20) + } + assert prompted == {15: False, 16: True, 19: True, 20: False} + + +uk_input_names = st.sampled_from( + [ + "savings_interest_income", + "dividend_income", + "property_income", + "pip_dl_category", + "is_disabled_for_benefits", + "gender", + "date_of_birth", + "current_education", + ] +) + + +@settings(max_examples=60, deadline=None) +@given( + adult_inputs=st.dictionaries( + uk_input_names, st.integers(min_value=1, max_value=10**6), max_size=5 + ), + household_inputs=st.dictionaries( + st.sampled_from( + ["rent", "savings", "tenure_type", "mortgage_interest_repayment"] + ), + st.integers(min_value=1, max_value=10**6), + max_size=4, + ), + num_children=st.integers(min_value=0, max_value=3), + wage=st.floats(min_value=0, max_value=10**6, allow_nan=False), +) +def test_uk_situation_holds_exactly_the_prompted_facts( + adult_inputs, household_inputs, num_children, wage +): + scenario = Scenario( + id="uk", + country="uk", + state="SCOTLAND", + filing_status=None, + adults=[ + Person(name="adult1", age=40, employment_income=wage, inputs=adult_inputs) + ], + children=[ + Person(name=f"child{i + 1}", age=5, employment_income=0.0) + for i in range(num_children) + ], + household_inputs=household_inputs, + year=2026, + ) + + situation = scenario.to_pe_uk_situation(prefix="s__") + + people = situation["people"] + assert set(people) == {f"s__{p.name}" for p in scenario.all_people} + adult = people["s__adult1"] + assert set(adult) == {"age", "employment_income_before_lsr", *adult_inputs} + assert adult["employment_income_before_lsr"] == {"2026": wage} + for key, value in adult_inputs.items(): + expected = int(value) if key == "date_of_birth" else value + assert adult[key] == {"2026": expected} + (benunit,) = situation["benunits"].values() + (household,) = situation["households"].values() + assert benunit["members"] == household["members"] == list(people) + assert set(household) == {"members", "region", *household_inputs} + assert household["region"] == {"2026": "SCOTLAND"} + + +def test_uk_situation_rejects_us_scenarios(): + scenario = Scenario( + id="us", + state="CA", + filing_status="single", + adults=[Person(name="adult1", age=40, employment_income=1.0)], + ) + with pytest.raises(ValueError, match="only supported for UK"): + scenario.to_pe_uk_situation() diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index 23f63be9..8c9b2eef 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -18,7 +18,7 @@ import pandas as pd import pytest -from policybench.config import MODELS, PROGRAMS +from policybench.config import MODELS, PROGRAMS, get_programs from policybench.eval_no_tools import ( NO_TOOLS_RESULT_COLUMNS, _build_resume_metadata, @@ -1540,3 +1540,59 @@ def test_supervised_run_keeps_policyengine_out_of_workers(tmp_path): (run_dir / POLICYENGINE_PROVENANCE_FILENAME).resolve() ) assert heartbeat["policyengine_provenance_recomputed"] == 0 + + +@pytest.fixture +def uk_manifest(tmp_path: Path) -> Path: + path = tmp_path / "uk_scenarios.csv" + scenarios = [ + Scenario( + id=f"scenario_{i:03d}", + country="uk", + state="WALES", + filing_status=None, + adults=[Person(name="adult1", age=35, employment_income=20_000)], + source_dataset="uk_calibrated_transfer_2025", + ) + for i in range(N_SCENARIOS) + ] + pd.DataFrame( + { + "scenario_id": [scenario.id for scenario in scenarios], + "scenario_json": [ + json.dumps(scenario_to_dict(scenario)) for scenario in scenarios + ], + } + ).to_csv(path, index=False) + return path + + +def test_uk_workers_request_the_uk_country_and_output_set( + uk_manifest, tmp_path, monkeypatch +): + supervisor = make_supervisor(uk_manifest, tmp_path) + launched = [] + + class FakePopen: + def __init__(self, cmd, **kwargs): + launched.append(cmd) + + monkeypatch.setattr(subprocess, "Popen", FakePopen) + supervisor._spawn(0) + + (cmd,) = launched + assert cmd[cmd.index("--country") + 1] == "uk" + assert supervisor._expected_outputs_for_scenario(0) == get_programs("uk") + assert supervisor.initial_request_variables == get_programs("uk") + assert supervisor.workload["output_set_sha256"] == hashlib.sha256( + "\n".join(sorted(get_programs("uk"))).encode("utf-8") + ).hexdigest() + + +def test_us_workload_fingerprint_is_unchanged_by_country_awareness( + manifest, tmp_path +): + supervisor = make_supervisor(manifest, tmp_path) + assert supervisor.workload["output_set_sha256"] == hashlib.sha256( + "\n".join(sorted(PROGRAMS)).encode("utf-8") + ).hexdigest() diff --git a/uv.lock b/uv.lock index 10b437c3..c294da04 100644 --- a/uv.lock +++ b/uv.lock @@ -2510,7 +2510,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.24" }, { name = "pandas", specifier = ">=2.0" }, { name = "policyengine", specifier = "==6.1.2" }, - { name = "policyengine-uk", specifier = "==2.89.0" }, + { name = "policyengine-uk", specifier = "==2.125.1" }, { name = "policyengine-us", specifier = "==2.38.6" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, @@ -2566,7 +2566,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.89.0" +version = "2.125.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2574,9 +2574,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e4/49/75c80cd9b1145d8cfd5efbdb534aca2c9d5c5722fef3fc8f4887831505ea/policyengine_uk-2.89.0.tar.gz", hash = "sha256:aea49956a350ec35c1e0def7cd28db324b055711ec5ab7f331395dca8d103341", size = 1216776, upload-time = "2026-06-11T13:06:04.826Z" } +sdist = { url = "https://files.pythonhosted.org/packages/64/3b/c7b4eaa2b80df7e520d0576399cf1c2cec6e29bdf4ecb0104496cfa0b946/policyengine_uk-2.125.1.tar.gz", hash = "sha256:96e0354c3e8d19cd9d4e7dc5ba6c3611f1305a589b63c5f065441df346c33ed5", size = 2112669, upload-time = "2026-10-10T00:21:19.345Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bb/5f/fd32a3f7314b02b3b6a8362d6b2869af5b51569301945c00ee1172e01914/policyengine_uk-2.89.0-py3-none-any.whl", hash = "sha256:72acae45a35a59cf7429bba69363601e624bfd643d50cd74b8d20d325497e9fd", size = 1999087, upload-time = "2026-06-11T13:06:02.819Z" }, + { url = "https://files.pythonhosted.org/packages/ec/3a/1d90bad4346d1a8c2715be211ed3d3d7f48917da0d90405b8257b7304091/policyengine_uk-2.125.1-py3-none-any.whl", hash = "sha256:a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a", size = 3305979, upload-time = "2026-10-10T00:21:17.194Z" }, ] [[package]] From 3543a45fb1b8d22ddcf5e15459eb28844014888a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 14:37:18 -0400 Subject: [PATCH 2/9] Format the supervisor UK tests Co-Authored-By: Claude Opus 5.5 --- tests/test_supervisor.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index 8c9b2eef..afb1a95f 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -1584,15 +1584,17 @@ def __init__(self, cmd, **kwargs): assert cmd[cmd.index("--country") + 1] == "uk" assert supervisor._expected_outputs_for_scenario(0) == get_programs("uk") assert supervisor.initial_request_variables == get_programs("uk") - assert supervisor.workload["output_set_sha256"] == hashlib.sha256( - "\n".join(sorted(get_programs("uk"))).encode("utf-8") - ).hexdigest() + assert ( + supervisor.workload["output_set_sha256"] + == hashlib.sha256( + "\n".join(sorted(get_programs("uk"))).encode("utf-8") + ).hexdigest() + ) -def test_us_workload_fingerprint_is_unchanged_by_country_awareness( - manifest, tmp_path -): +def test_us_workload_fingerprint_is_unchanged_by_country_awareness(manifest, tmp_path): supervisor = make_supervisor(manifest, tmp_path) - assert supervisor.workload["output_set_sha256"] == hashlib.sha256( - "\n".join(sorted(PROGRAMS)).encode("utf-8") - ).hexdigest() + assert ( + supervisor.workload["output_set_sha256"] + == hashlib.sha256("\n".join(sorted(PROGRAMS)).encode("utf-8")).hexdigest() + ) From 94d524d06ca0dcbd6a9b6a903f506b5be58bc09a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:09:24 -0400 Subject: [PATCH 3/9] Let the reference adversary audit the UK leg consensus-flags and adversary-prepare take --country; load_payload reads a country's payload (load_us_payload keeps its behaviour). UK cases get UK prompts: fiscal year 2026-27, legislation.gov.uk and GOV.UK sources, pounds, the region, and a 2026-10-10 freeze (policyengine-uk 2.125.1), with the schema's freeze date to match. All 62 committed US cases re-render byte for byte and the US schemas are unchanged. Also pin policyengine-uk 2.125.1, the latest release. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 12 ++ policybench/cli.py | 24 +++- policybench/consensus.py | 33 +++-- policybench/reference_adversary.py | 194 +++++++++++++++++++++++++++-- tests/test_consensus.py | 22 ++++ tests/test_reference_adversary.py | 72 +++++++++++ 6 files changed, 329 insertions(+), 28 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index 9dfe007d..aa19bb3d 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -115,6 +115,18 @@ The defaults round answers to the nearest dollar, which merges answers such as model whose own answer is within the tolerance of an amount reference counts as exact and never toward a wrong cluster, even when its rounded key misses. +### The UK leg + +`consensus-flags` and `adversary-prepare` take `--country uk`. They read the +`uk/data.json` that `export-full-run --country uk` writes, or the `uk` entry +of a combined payload. UK cases get UK prompts: the 2026-27 fiscal year, +legislation.gov.uk and GOV.UK as primary sources, pounds, the region, and a +2026-10-10 law freeze (policyengine-uk 2.125.1). The UK board has six models, +so pass thresholds that fit it, for example `--min-models 2 --top-k 3 +--min-top 2 --zero-cluster-min-models 2`. A UK derivation is the +engine's own computation log, pruned to the steps that carry a value, in +`uk_case_reference_explanations.csv`. US prompts are unchanged byte for byte. + ## Two stages - **Stage 1, law first.** The judge sees the household prompt, the output's diff --git a/policybench/cli.py b/policybench/cli.py index b2c8128f..0d8113b3 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -594,6 +594,12 @@ def main(): help="Frozen run payload (data.json.gz) or a release dashboard-data.json", ) consensus_parser.add_argument("--output", required=True, help="Report JSON path") + consensus_parser.add_argument( + "--country", + choices=["us", "uk"], + default="us", + help="Country payload to read (default us)", + ) consensus_parser.add_argument( "--prototype", action="store_true", @@ -624,6 +630,12 @@ def main(): help="Write two-stage reference-adversary prompts for consensus-flagged cells", ) adversary_prepare_parser.add_argument("--payload", required=True) + adversary_prepare_parser.add_argument( + "--country", + choices=["us", "uk"], + default="us", + help="Country payload to read (default us)", + ) adversary_prepare_parser.add_argument( "--flags", required=True, help="consensus-flags report JSON" ) @@ -1424,7 +1436,7 @@ def main(): ConsensusParams, consensus_report, file_sha256, - load_us_payload, + load_payload, ) params = PROTOTYPE_PARAMS if args.prototype else ConsensusParams() @@ -1444,7 +1456,7 @@ def main(): params = replace(params, **overrides) payload_path = Path(args.payload) report = consensus_report( - load_us_payload(payload_path), + load_payload(payload_path, args.country), params, source=str(payload_path), source_sha256=file_sha256(payload_path), @@ -1457,7 +1469,7 @@ def main(): ) elif args.command == "adversary-prepare": - from policybench.consensus import load_us_payload + from policybench.consensus import load_payload from policybench.reference_adversary import ( build_adversary_cases, load_derivations, @@ -1472,12 +1484,14 @@ def main(): skipped[key] = str(cell.get("covered_by", "")) kept = [f for f in flags if (f["scenario_id"], f["variable"]) not in skipped] derivations = ( - load_derivations(Path(args.annotations_dir)) + load_derivations(Path(args.annotations_dir), args.country) if args.annotations_dir else None ) cases = build_adversary_cases( - load_us_payload(Path(args.payload)), kept, derivations=derivations + load_payload(Path(args.payload), args.country), + kept, + derivations=derivations, ) prepare_adversary(Path(args.adversary_dir), cases) print( diff --git a/policybench/consensus.py b/policybench/consensus.py index 99ba6edb..7674e326 100644 --- a/policybench/consensus.py +++ b/policybench/consensus.py @@ -123,13 +123,19 @@ def to_dict(self) -> dict[str, Any]: PROTOTYPE_PARAMS = ConsensusParams(answer_rounding="truncate", binary_outputs="skip") -def load_us_payload(path: Path | str) -> dict: - """Read a US dashboard payload from ``.json`` or ``.json.gz``. +PAYLOAD_COUNTRIES = ("us", "uk") + + +def load_payload(path: Path | str, country: str = "us") -> dict: + """Read one country's dashboard payload from ``.json`` or ``.json.gz``. - Accepts both the per-country payload frozen under ``paper/snapshot`` and - the combined ``{"countries": {"us": ...}}`` shape of a dashboard release - asset, and returns the US payload either way. + Accepts both a per-country payload (frozen under ``paper/snapshot``, or + ``//data.json`` from ``export-full-run``) and the combined + ``{"countries": {...}}`` shape of a dashboard release asset, and returns + that country's payload either way. """ + if country not in PAYLOAD_COUNTRIES: + raise ValueError(f"country must be one of {PAYLOAD_COUNTRIES}: {country!r}") path = Path(path) raw = path.read_bytes() if raw[:2] == b"\x1f\x8b": @@ -139,17 +145,22 @@ def load_us_payload(path: Path | str) -> dict: raise ValueError(f"{path}: payload is not a JSON object") if "countries" in payload: countries = payload["countries"] - if not isinstance(countries, dict) or "us" not in countries: - raise ValueError(f"{path}: combined payload has no 'us' country") - payload = countries["us"] - country = payload.get("country") - if country is not None and country != "us": - raise ValueError(f"{path}: payload is for country {country!r}, not 'us'") + if not isinstance(countries, dict) or country not in countries: + raise ValueError(f"{path}: combined payload has no {country!r} country") + payload = countries[country] + found = payload.get("country") + if found is not None and found != country: + raise ValueError(f"{path}: payload is for country {found!r}, not {country!r}") if "scenarioPredictions" not in payload or "modelStats" not in payload: raise ValueError(f"{path}: payload lacks scenarioPredictions or modelStats") return payload +def load_us_payload(path: Path | str) -> dict: + """Read a US dashboard payload (see :func:`load_payload`).""" + return load_payload(path, "us") + + def file_sha256(path: Path | str) -> str: """The sha256 of a file's bytes, as lowercase hex.""" digest = hashlib.sha256() diff --git a/policybench/reference_adversary.py b/policybench/reference_adversary.py index b30702ca..5dfd5740 100644 --- a/policybench/reference_adversary.py +++ b/policybench/reference_adversary.py @@ -68,6 +68,10 @@ # reference_audit/2026-09-28/fixes/latest_final.py encodes law published before # this date. REFERENCE_FREEZE = "2026-07-03" +# The UK leg's references come from policyengine-uk 2.125.1, released on +# 2026-10-10; the law it encodes is the reference's law for the UK. +REFERENCE_FREEZES = {"us": REFERENCE_FREEZE, "uk": "2026-10-10"} +UK_FISCAL_YEAR = f"{TAX_YEAR}-{str(TAX_YEAR + 1)[-2:]}" VERDICTS = ( "reference_holds", @@ -272,6 +276,24 @@ } +def schema_for_country(schema: dict, country: str) -> dict: + """``schema`` with its citations' freeze date set for ``country``. + + Only the ``pre_freeze`` description differs, so every country's outputs + validate against the same structure. + """ + freeze = REFERENCE_FREEZES.get(country) + if freeze is None: + raise ValueError(f"no reference freeze for country {country!r}") + if freeze == REFERENCE_FREEZE: + return schema + text = json.dumps(schema).replace( + f"True if published before {REFERENCE_FREEZE}, ", + f"True if published before {freeze}, ", + ) + return json.loads(text) + + def canonical_json(document: Any) -> str: """The one serialization of a judge output this module writes and hashes.""" return json.dumps(document, indent=2, sort_keys=True) @@ -801,6 +823,139 @@ def build_adversary_cases( Return only the JSON object the schema asks for.""" +_UK_FREEZE = REFERENCE_FREEZES["uk"] +_UK_YEAR_SPAN = ( + f"the UK fiscal year {UK_FISCAL_YEAR} " + f"(6 April {TAX_YEAR} to 5 April {TAX_YEAR + 1})" +) + +_UK_STAGE1_HEADER = f"""\ +You are a reference adversary for a UK tax-and-benefit benchmark. Each \ +benchmark question gives a household and asks for policy quantities for \ +{_UK_YEAR_SPAN}. A microsimulation engine produced the reference answer. On \ +the question below, one or more AI models answering from memory, without \ +tools, gave an answer other than the reference. Agreement among models proves \ +nothing, and neither does the engine. Your job is to find out what the law \ +gives. + +This is stage 1 of 2. In this stage you work the answer out yourself from \ +primary law, before you see anything about how the engine computed its value. + +How to work: +1. Use web search and web fetch to find and read primary law in force for \ +{UK_FISCAL_YEAR}: Acts and statutory instruments on legislation.gov.uk \ +(including the {TAX_YEAR} up-rating orders and rates regulations), the rates \ +and thresholds HMRC and DWP publish on GOV.UK for {UK_FISCAL_YEAR}, HMRC \ +manuals, and DWP's Advice for decision making. Prefer legislation.gov.uk and \ +gov.uk. +2. Apply the household prompt's conventions exactly as the models were told \ +them: treat any unlisted numeric input as 0 and any other unlisted fact, \ +boolean or status as false; assume program take-up when required; do not \ +infer unlisted income, expenses, assets, benefit receipt or rent. +3. Read the output definition literally. It decides which people, benefit \ +units, taxes or benefits the number covers. Where it lists components, decide \ +for each candidate amount whether the definition includes it. +4. Cite every rule you rely on: the source, a pinpoint (section, regulation, \ +paragraph, table or page), the URL you read, the publication or effective \ +date, and a short verbatim quote. The reference's law was frozen on \ +{_UK_FREEZE}: set pre_freeze to true if the source was published before that \ +date, false if after, and null if you cannot tell. If an amount for \ +{UK_FISCAL_YEAR} (a rate, threshold, allowance or benefit amount) had not been \ +published before {_UK_FREEZE}, say so in computation and say what had been \ +published. +5. {_SOURCE_RULES} +6. Say which answer the law supports: "reference", "consensus", "neither", or \ +"both_readings" (the reference under one reasonable reading of the definition \ +or the facts, the consensus under another). +7. In definition_reading, say how you read the output definition for this \ +household. In ambiguity, describe any second reading that the definition or \ +the household prompt genuinely admits, and the answer it gives; use "" if \ +there is none. +8. independent_answer is your own number, an annual amount in pounds. Use \ +null only when the stated facts and the law leave it genuinely undetermined, \ +and say why in ambiguity. + +Return only the JSON object the schema asks for.""" + +_UK_STAGE2_HEADER = f"""\ +You are a reference adversary for a UK tax-and-benefit benchmark. Each \ +benchmark question gives a household and asks for policy quantities for \ +{_UK_YEAR_SPAN}. A microsimulation engine produced the reference answer. On \ +the question below, one or more AI models answering from memory, without \ +tools, gave an answer other than the reference. + +This is stage 2 of 2: reconciliation. In stage 1, a judge who had not seen \ +how the engine derived the reference worked the question from primary law. \ +Its result appears below exactly as it was recorded, with its sha256. Stage 1 \ +is frozen and you may not revise it. If the engine derivation or the law \ +shows that stage 1 erred (it misread a fact, missed or misapplied a rule, or \ +used the wrong year's amount), say so in stage1_error and name the error. Do \ +not silently change course: if your independent_answer, or your view of which \ +answer the law supports, differs from stage 1's, stage1_error must say why. \ +Use "" for stage1_error only when stage 1 stands. + +Only now do you see how the engine derived the reference. The derivation is \ +the engine's own computation log for this output and household: each variable \ +it computed for {UK_FISCAL_YEAR} with its value (annual amounts in pounds; a \ +variable over several people lists one value per person), indented under the \ +variable that used it, then every parameter it read with its value (a \ +parameter's own period, such as week or month, applies). Compare each step \ +with the law and with the output definition, and decide: +- reference_holds: the reference is what the law and the output definition \ +give on the stated facts. +- reference_wrong: the engine misapplies the law on the stated facts, or \ +uses an amount or rule the law had not published (for example a \ +{UK_FISCAL_YEAR} amount the engine projected itself where the government had \ +published a different one, or none, before {_UK_FREEZE}). +- definition_mismatch: the engine computes something other than what the \ +output definition describes (it includes or leaves out people, taxes or \ +benefits that the definition covers or excludes). +- prompt_ambiguous: the stated facts or the definition genuinely admit more \ +than one answer (for example the answer turns on an input the prompt does not \ +list). + +engine_step_at_issue names the derivation step that departs from the law or \ +the definition; use "" when the reference holds. + +suggested_adjudication: +- affirmed: the reference holds. +- regenerated: the reference should be recomputed under a stated convention \ +or a corrected input; the output stays scored. +- engine_defect: the engine misapplies the law on the stated facts. +- unlisted_input: the answer turns on an input the prompt does not list. +- later_law: the reference rests on law or an amount published after \ +{_UK_FREEZE}, or never published. +- definition_exclusion: the engine's quantity does not match the output \ +definition, so the output should be excluded. +- none: you suggest nothing. +Pair them this way: reference_holds with affirmed; reference_wrong with \ +engine_defect, later_law or regenerated; definition_mismatch with \ +definition_exclusion or regenerated; prompt_ambiguous with unlisted_input or \ +definition_exclusion. "none" goes with any verdict. + +reference_value is the engine reference below. consensus_value is the answer \ +of the models you weighed (null if none applies). + +Cite the law you rely on as in stage 1: source, pinpoint, URL, date, \ +pre_freeze and a short quote. The engine derivation is not a citation. \ +{_SOURCE_RULES} + +Your verdict changes no score. A developer adjudicates every case you do not \ +return as reference_holds. + +Return only the JSON object the schema asks for.""" + +_STAGE1_HEADERS = {"us": _STAGE1_HEADER, "uk": _UK_STAGE1_HEADER} +_STAGE2_HEADERS = {"us": _STAGE2_HEADER, "uk": _UK_STAGE2_HEADER} + + +def _country_template(templates: dict, country: str) -> str: + try: + return templates[country] + except KeyError: + raise ValueError(f"no reference-adversary prompt for country {country!r}") + + def _block(title: str, text: str) -> str: return f"----- BEGIN {title} -----\n{text}\n----- END {title} -----" @@ -809,7 +964,7 @@ def _reference_text(case: AdversaryCase) -> str: value = case.reference_value if case.metric_type == "binary": return f"{_number(value)} ({'eligible' if value else 'not eligible'})" - rounded = f"${value:,.2f}" + rounded = f"{'£' if case.country == 'uk' else '$'}{value:,.2f}" if round(value, 2) == value: return rounded return f"{rounded} (engine output {_number(value)})" @@ -817,14 +972,25 @@ def _reference_text(case: AdversaryCase) -> str: def _case_sections(case: AdversaryCase) -> list[str]: """The case as both stages show it; never the derivation.""" - kind = ( - "an eligibility flag, 1 or 0" - if case.metric_type == "binary" - else "an annual dollar amount" - ) + if case.country == "uk": + kind = "an annual amount in pounds" + header = [ + f"UK FISCAL YEAR: {UK_FISCAL_YEAR} REFERENCE LAW FROZEN: " + f"{REFERENCE_FREEZES['uk']}", + f"REGION: {case.state or 'see the household prompt'}", + ] + else: + kind = ( + "an eligibility flag, 1 or 0" + if case.metric_type == "binary" + else "an annual dollar amount" + ) + header = [ + f"TAX YEAR: {TAX_YEAR} REFERENCE LAW FROZEN: {REFERENCE_FREEZE}", + f"STATE: {case.state or 'see the household prompt'}", + ] lines = [ - f"TAX YEAR: {TAX_YEAR} REFERENCE LAW FROZEN: {REFERENCE_FREEZE}", - f"STATE: {case.state or 'see the household prompt'}", + *header, f"OUTPUT: {case.variable} ({kind})", f"OUTPUT DEFINITION, as the models saw it: {case.definition}", "", @@ -856,7 +1022,8 @@ def _case_sections(case: AdversaryCase) -> list[str]: def render_stage1_prompt(case: AdversaryCase) -> str: """The law-first prompt: everything about the case except the derivation.""" - return "\n".join([_STAGE1_HEADER, "", *_case_sections(case)]) + header = _country_template(_STAGE1_HEADERS, case.country) + return "\n".join([header, "", *_case_sections(case)]) def render_stage2_prompt(case: AdversaryCase, stage1: dict, stage1_sha256: str) -> str: @@ -869,7 +1036,7 @@ def render_stage2_prompt(case: AdversaryCase, stage1: dict, stage1_sha256: str) derivation = case.derivation or "(no derivation was recorded for this case)" return "\n".join( [ - _STAGE2_HEADER, + _country_template(_STAGE2_HEADERS, case.country), "", *_case_sections(case), "", @@ -921,11 +1088,14 @@ def prepare_adversary( if duplicates: raise ValueError(f"duplicate adversary cases: {duplicates}") adversary_dir.mkdir(parents=True, exist_ok=True) + countries = sorted({case.country for case in cases}) or ["us"] + if len(countries) > 1: + raise ValueError(f"adversary cases mix countries: {countries}") (adversary_dir / STAGE1_SCHEMA_FILE).write_text( - json.dumps(STAGE1_SCHEMA, indent=2) + "\n" + json.dumps(schema_for_country(STAGE1_SCHEMA, countries[0]), indent=2) + "\n" ) (adversary_dir / VERDICT_SCHEMA_FILE).write_text( - json.dumps(VERDICT_SCHEMA, indent=2) + "\n" + json.dumps(schema_for_country(VERDICT_SCHEMA, countries[0]), indent=2) + "\n" ) cases_root = adversary_dir / "cases" derivations_root = adversary_dir / "derivations" diff --git a/tests/test_consensus.py b/tests/test_consensus.py index d1b42991..4c98526a 100644 --- a/tests/test_consensus.py +++ b/tests/test_consensus.py @@ -887,3 +887,25 @@ def test_models_exact_agrees_with_the_board_scorer(consensus): cell = _frozen()["scenarioPredictions"][flag["scenario_id"]][flag["variable"]] hits = sum(entry["exact"] == 100.0 for entry in cell.values()) assert flag["models_exact"] == hits, (flag["scenario_id"], flag["variable"]) + + +def test_load_payload_reads_the_requested_country(tmp_path): + from policybench.consensus import load_payload + + us = make_payload( + ["a", "b"], {("scenario_001", "snap"): (5.0, True, {"a": 1.0, "b": 1.0})} + ) + uk = dict(us, country="uk") + us = dict(us, country="us") + release = tmp_path / "release.json" + release.write_text(json.dumps({"countries": {"us": us, "uk": uk}})) + uk_only = tmp_path / "uk.json" + uk_only.write_text(json.dumps(uk)) + + assert load_payload(release, "uk") == uk + assert load_payload(release, "us") == us + assert load_payload(uk_only, "uk") == uk + with pytest.raises(ValueError, match="not 'us'"): + load_payload(uk_only, "us") + with pytest.raises(ValueError, match="country must be one of"): + load_payload(uk_only, "fr") diff --git a/tests/test_reference_adversary.py b/tests/test_reference_adversary.py index 0f40004b..45c4352e 100644 --- a/tests/test_reference_adversary.py +++ b/tests/test_reference_adversary.py @@ -1791,3 +1791,75 @@ def test_frozen_run_cases_are_blind_in_stage1(tmp_path: Path): raw = json.load(handle) cell = raw["scenarioPredictions"]["scenario_123"][TAX] assert next(iter(cell.values()))["referenceExplanation"] + + +def _uk_case(cases: list[AdversaryCase]) -> AdversaryCase: + from dataclasses import replace + + return replace( + cases[0], + case_id="uk__scenario_001__income_tax", + country="uk", + variable="income_tax", + state="WALES", + reference_value=253.03, + ) + + +def test_uk_cases_render_uk_law_and_currency(cases): + from policybench.reference_adversary import REFERENCE_FREEZES, render_stage2_prompt + + case = _uk_case(cases) + stage1 = render_stage1_prompt(case) + stage2 = render_stage2_prompt(case, _stage1(), "0" * 64) + + for prompt in (stage1, stage2): + assert prompt.startswith("You are a reference adversary for a UK ") + assert "UK FISCAL YEAR: 2026-27 REFERENCE LAW FROZEN: 2026-10-10" in prompt + assert "REGION: WALES" in prompt + assert "ENGINE REFERENCE VALUE: £253.03" in prompt + assert "US tax-and-benefit" not in prompt + assert "STATE:" not in prompt + assert "legislation.gov.uk" in stage1 + assert "computation log" in stage2 + assert REFERENCE_FREEZES == {"us": "2026-07-03", "uk": "2026-10-10"} + + +def test_us_cases_keep_the_us_prompt(cases): + from policybench.reference_adversary import _STAGE1_HEADER + + prompt = render_stage1_prompt(cases[0]) + assert prompt.startswith(_STAGE1_HEADER) + assert "TAX YEAR: 2026 REFERENCE LAW FROZEN: 2026-07-03" in prompt + + +def test_unknown_country_has_no_adversary_prompt(cases): + from dataclasses import replace + + with pytest.raises(ValueError, match="no reference-adversary prompt"): + render_stage1_prompt(replace(cases[0], country="fr")) + + +def test_prepare_writes_the_uk_freeze_into_the_uk_schema(cases, tmp_path): + from policybench.reference_adversary import ( + STAGE1_SCHEMA, + prepare_adversary, + schema_for_country, + ) + + prepare_adversary(tmp_path / "uk", [_uk_case(cases)]) + written = json.loads((tmp_path / "uk" / "schema_stage1.json").read_text()) + assert "True if published before 2026-10-10" in json.dumps(written) + assert "2026-07-03" not in json.dumps(written) + # Only the description differs, so outputs validate the same way. + assert json.dumps(written).replace("2026-10-10", "2026-07-03") == json.dumps( + STAGE1_SCHEMA + ) + assert schema_for_country(STAGE1_SCHEMA, "us") == STAGE1_SCHEMA + + +def test_prepare_refuses_mixed_countries(cases, tmp_path): + from policybench.reference_adversary import prepare_adversary + + with pytest.raises(ValueError, match="mix countries"): + prepare_adversary(tmp_path / "mixed", [cases[0], _uk_case(cases)]) From 70150a9ada522b72cc62368eec5f5588b115e2a6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:32:53 -0400 Subject: [PATCH 4/9] Store UK prompt inputs as the whole numbers the prompt shows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prompt formats every UK amount with no decimals, but the situation used the unrounded reference-period values (£15,095.16 shown as £15,095), so 150 of the internal run's 700 references sat up to £0.99 from the stated facts. Round each UK numeric input as the formatter does (half to even) when building scenarios; an amount shown as £0 becomes unlisted. For the frozen internal run, a rounded copy of the manifest renders all 100 prompts byte for byte as the models saw them, and its references leave the 59 consensus flags unchanged. Property test: rounding never changes what the prompt displays for an amount that stays in the prompt. Co-Authored-By: Claude Opus 5.5 --- policybench/scenarios.py | 16 +++++++++++++--- tests/test_scenarios.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 38590e17..b69eb3d1 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -1428,6 +1428,16 @@ def scenarios_from_cps_frame( return scenarios +def round_uk_prompt_number(value: float) -> float: + """A UK numeric input as the prompt shows it: to the whole unit. + + The prompt formats every UK amount, hour count and date with no decimals + (Python's round-half-even, as the formatter rounds), so storing the + rounded value makes the reference use exactly the stated fact. + """ + return float(round(float(value))) + + def _uk_promptable_value(value: Any) -> Any | None: if pd.isna(value): return None @@ -1439,10 +1449,10 @@ def _uk_promptable_value(value: Any) -> Any | None: return None return cleaned try: - numeric = float(value) + numeric = round_uk_prompt_number(value) except (TypeError, ValueError): return value - if abs(numeric) <= 1e-6: + if numeric == 0: return None return numeric @@ -1490,7 +1500,7 @@ def _build_uk_person(row: pd.Series, label: str) -> Person: continue value = pd.to_numeric(row[column], errors="coerce") if not pd.isna(value): - employment_income = float(value) + employment_income = round_uk_prompt_number(value) break return Person( name=label, diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index f2611e61..e1c66285 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1477,3 +1477,32 @@ def test_uk_situation_rejects_us_scenarios(): ) with pytest.raises(ValueError, match="only supported for UK"): scenario.to_pe_uk_situation() + + +@settings(max_examples=500, deadline=None) +@given( + value=st.floats( + min_value=-1e7, max_value=1e7, allow_nan=False, allow_infinity=False + ) +) +def test_uk_rounding_keeps_the_displayed_number(value): + from policybench.prompts import _format_input_line + + rounded = scenarios_module.round_uk_prompt_number(value) + assert rounded == int(rounded) + if rounded == 0: + # Shown as £0 (or £-0) before; now unlisted, which the prompt reads as 0. + assert scenarios_module._uk_promptable_value(value) is None + return + # The prompt shows the rounded value exactly as it showed the raw one, so + # the reference uses the number the models read. + assert _format_input_line("savings", value, country="uk") == _format_input_line( + "savings", rounded, country="uk" + ) + + +def test_uk_promptable_values_are_whole_and_drop_amounts_shown_as_zero(): + assert scenarios_module._uk_promptable_value(15_095.16) == 15_095.0 + assert scenarios_module._uk_promptable_value(-107_891.4) == -107_891.0 + assert scenarios_module._uk_promptable_value(0.4) is None + assert scenarios_module._uk_promptable_value(0.6) == 1.0 From 140e830a05c452dadd13d620b4124a9300ecf2e1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 08:53:11 -0400 Subject: [PATCH 5/9] State UK relationships, locality and education entry in prompt and reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the review of #212: - Relationships: the listed adults go in as claimant and partner and every child as neither (is_claimant_or_partner), and a couple as not married, matching the prompt, which now says so. PE-UK otherwise presumed a 37/18 pair to be parent and child and presumed every couple married, giving Marriage Allowance the prompt never states. On the frozen run this moves ten income tax references by £252 and resolves five consensus cells. - Locality: a private renter gets a Broad Rental Market Area, the area of the region's largest city, stored and stated in the prompt. PE-UK reads the LHA from the BRMA alone and defaults everyone to Maidstone. - Education: someone in non-advanced education is stated (and supplied) to be in full-time education begun before 19; PE-UK's default entry age (1000) failed the qualifying-young-person entry condition. - LCWRA wording: the assessment, with protected status conditional on UC entitlement, so pension-age adults are not told they hold an award. - Boundary: to_pe_uk_situation rejects tax-unit or benefit inputs and person fields the prompt filters out. - Tests: the situation property test uses valid typed values and checks each input against the rendered prompt; new tests for the boundary, the renter BRMA, and (slow, real engine) an unmarried 37/18 couple. - docs/audit.md names each country's freeze; the benchmark card describes the new conventions. Co-Authored-By: Claude Opus 5.5 --- docs/audit.md | 5 +- docs/benchmark_card.md | 11 +++ policybench/prompts.py | 12 ++- policybench/scenarios.py | 63 ++++++++++++- tests/test_eval_no_tools.py | 9 +- tests/test_scenarios.py | 173 ++++++++++++++++++++++++++++++------ 6 files changed, 236 insertions(+), 37 deletions(-) diff --git a/docs/audit.md b/docs/audit.md index aa19bb3d..cf244990 100644 --- a/docs/audit.md +++ b/docs/audit.md @@ -134,8 +134,9 @@ engine's own computation log, pruned to the steps that carry a value, in member's model id, answer and explanation, and the reference value. It does not see the engine derivation. It works the answer from primary law (statutes, regulations, agency publications, forms and instructions), cites - each rule with its publication date and whether it predates the 2026-07-03 - reference freeze, and says which answer the law supports. + each rule with its publication date and whether it predates the country's + reference freeze (US 2026-07-03, UK 2026-10-10), and says which answer the + law supports. - The Claude runner gives it only web tools (no file access) and denies WebFetch on the PolicyBench, PolicyEngine and GitHub domains (`BLOCKED_DOMAINS`). WebSearch has no deny rule, so its results reach the diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index d0654cb5..408f3744 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -432,6 +432,17 @@ so by listing none. Since October 2026 the path works as follows. status of anyone aged 16 to 19 (qualifying young person rules), and an adult's limited capability for work and work-related activity for Universal Credit, held since before 6 April 2026. +- **Relationships and locality stated.** The prompt says two listed adults are + a couple who are not married or in a civil partnership, and the reference + supplies exactly that (claimant and partner flags, no marriage), so neither + PE-UK's presumed-child rule nor its presumption that couples are married + applies. A private renter's prompt states a Broad Rental Market Area: the + area of the region's largest city, since the transfer data records none + and PE-UK otherwise reads Maidstone's Local Housing Allowance everywhere. + Someone in non-advanced education is stated to have begun it before 19. +- **Whole-number facts.** Every UK amount is stored as the whole number the + prompt shows, so the reference uses the stated figure, not an unrounded + uprated one. - **Engine-version guards.** Reference generation stops if the artifact stores a variable the installed PolicyEngine UK does not define, or if a derived fact the prompt states is renamed, so a model release cannot silently drop diff --git a/policybench/prompts.py b/policybench/prompts.py index 6f719684..54d51c90 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -159,6 +159,7 @@ "weekly_hours_worked": "hours worked per week", "hours_worked": "annual hours worked", "current_education": "current education", + "brma": "Broad Rental Market Area (for the Local Housing Allowance)", "date_of_birth": "date of birth", "workers_compensation": "workers' compensation", "capital_gains_before_response": "capital gains", @@ -227,8 +228,9 @@ # the element before 6 April 2026 when a household is simulated on its own. UK_DISABILITY_LABELS = { "adult": ( - "has limited capability for work and work-related activity (LCWRA) " - "for Universal Credit, held since before 6 April 2026" + "assessed as having limited capability for work and work-related " + "activity (LCWRA); if entitled to Universal Credit, the award has " + "included the LCWRA element since before 6 April 2026" ), "child": "is disabled (lower rate disabled child addition in Universal Credit)", } @@ -261,7 +263,8 @@ def _format_yyyymmdd(value) -> str: "NOT_IN_EDUCATION": "not in education or training", "UPPER_SECONDARY": "secondary school", "POST_SECONDARY": ( - "non-advanced education (school or college, below higher-education level)" + "full-time non-advanced education (school or college, below " + "higher-education level), begun before age 19" ), "TERTIARY": "advanced (higher) education", } @@ -430,7 +433,8 @@ def describe_household(scenario: Scenario) -> str: [ "Household structure:", "- all listed people live together in one UK benefit unit", - "- if two adults are listed, Adult 1 and Adult 2 are a couple", + "- if two adults are listed, Adult 1 and Adult 2 are a couple, " + "not married or in a civil partnership", ( "- children and qualifying young people are dependents, " "not partners" diff --git a/policybench/scenarios.py b/policybench/scenarios.py index b69eb3d1..e4e8ae5c 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -348,6 +348,32 @@ # Below 16 every person is a child; from 20 no one is a qualifying young person. UK_EDUCATION_PROMPT_AGES = range(16, 20) +# A Broad Rental Market Area per region for private renters: the area of the +# region's largest city. PE-UK reads the Local Housing Allowance from the BRMA +# alone and defaults every household to Maidstone, whatever its region; the +# transfer data records none. The prompt states the area it uses. +UK_REGION_BRMA = { + "NORTH_EAST": "TYNESIDE", + "NORTH_WEST": "CENTRAL_GREATER_MANCHESTER", + "YORKSHIRE": "LEEDS", + "EAST_MIDLANDS": "NOTTINGHAM", + "WEST_MIDLANDS": "BIRMINGHAM", + "EAST_OF_ENGLAND": "CAMBRIDGE", + "LONDON": "INNER_NORTH_LONDON", + "SOUTH_EAST": "MAIDSTONE", + "SOUTH_WEST": "BRISTOL", + "WALES": "CARDIFF", + "SCOTLAND": "GREATER_GLASGOW", + "NORTHERN_IRELAND": "BELFAST", +} +UK_PRIVATE_RENT_TENURE = "RENT_PRIVATELY" + +# The age a prompted person in non-advanced education began it. The data +# records none, and PE-UK's default (1000) would fail every +# qualifying-young-person entry condition; the prompt states "began before +# age 19". +UK_EDUCATION_ENTRY_AGE = 16 + UK_NON_PROMPTABLE_SENTINELS = { "", "NONE", @@ -609,23 +635,49 @@ def to_pe_uk_situation(self, prefix: str = "") -> dict: Each listed person, the region and the household inputs go in unchanged; nothing else from the source record does, so the reference is a function of the prompt. The benchmark keeps households with one - benefit unit, so every person joins that unit. ``prefix`` namespaces + benefit unit, so every person joins that unit. The prompt states the + relationships, so they go in explicitly rather than through PE-UK's + presumptions: the listed adults are the claimant and partner, every + child is neither, and a couple is not married (the prompt states no + marriage, and unlisted statuses are false). ``prefix`` namespaces entity keys when several scenarios share one simulation. + + Raises ``ValueError`` for inputs the prompt cannot show: tax-unit or + benefit inputs, or person fields the renderer filters out. """ if self.country != "uk": raise ValueError("to_pe_uk_situation is only supported for UK scenarios.") + if self.tax_unit_inputs or self.spm_unit_inputs: + raise ValueError( + f"{self.id}: UK scenarios cannot carry tax-unit or benefit inputs." + ) people = {} for person in self.all_people: + hidden = sorted( + k for k in person.inputs if is_excluded_prompt_input_name(k) + ) + if hidden: + raise ValueError( + f"{self.id} {person.name}: inputs the prompt does not show: " + f"{hidden}" + ) person_data = { "age": self._yearize(person.age), # The stored wage leaf; employment_income adds the (zero) # labour-supply response to it, as in the microsimulation. "employment_income_before_lsr": self._yearize(person.employment_income), + "is_claimant_or_partner": self._yearize( + person.name.startswith("adult") + ), } for key, value in person.inputs.items(): if key == "date_of_birth": value = int(value) person_data[key] = self._yearize(value) + if person.inputs.get("current_education") == "POST_SECONDARY": + person_data["age_started_or_accepted_current_education_or_training"] = ( + self._yearize(UK_EDUCATION_ENTRY_AGE) + ) people[f"{prefix}{person.name}"] = person_data members = list(people) household_data = { @@ -636,7 +688,12 @@ def to_pe_uk_situation(self, prefix: str = "") -> dict: household_data[key] = self._yearize(value) return { "people": people, - "benunits": {f"{prefix}benunit": {"members": members}}, + "benunits": { + f"{prefix}benunit": { + "members": members, + "is_married": self._yearize(False), + } + }, "households": {f"{prefix}household": household_data}, } @@ -1490,6 +1547,8 @@ def _extract_uk_household_inputs(row: pd.Series) -> dict[str, Any]: promptable = _uk_promptable_value(value) if promptable is not None: inputs[col] = promptable + if inputs.get("tenure_type") == UK_PRIVATE_RENT_TENURE: + inputs["brma"] = UK_REGION_BRMA[str(row["region"])] return inputs diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index c0b39159..318b6fd0 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -4006,8 +4006,9 @@ def test_uk_prompt_states_role_specific_disability_and_birth_facts(): prompt = make_no_tools_batch_prompt(scenario, ["universal_credit"]) assert ( - "- has limited capability for work and work-related activity (LCWRA) " - "for Universal Credit, held since before 6 April 2026" + "- assessed as having limited capability for work and work-related " + "activity (LCWRA); if entitled to Universal Credit, the award has " + "included the LCWRA element since before 6 April 2026" ) in prompt assert ( "- is disabled (lower rate disabled child addition in Universal Credit)" @@ -4017,8 +4018,8 @@ def test_uk_prompt_states_role_specific_disability_and_birth_facts(): assert "- gender: Female" in prompt assert "- annual hours worked: 1,820" in prompt assert ( - "- current education: non-advanced education (school or college, below " - "higher-education level)" + "- current education: full-time non-advanced education (school or " + "college, below higher-education level), begun before age 19" ) in prompt assert "- current education: advanced (higher) education" in prompt diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index e1c66285..6343d8d5 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1403,38 +1403,40 @@ def person_row(age): assert prompted == {15: False, 16: True, 19: True, 20: False} -uk_input_names = st.sampled_from( - [ - "savings_interest_income", - "dividend_income", - "property_income", - "pip_dl_category", - "is_disabled_for_benefits", - "gender", - "date_of_birth", - "current_education", - ] -) +UK_PROMPTED_PERSON_INPUTS = { + "savings_interest_income": st.integers(min_value=1, max_value=10**6), + "dividend_income": st.integers(min_value=1, max_value=10**6), + "property_income": st.integers(min_value=-(10**5), max_value=10**6).filter(bool), + "pip_dl_category": st.sampled_from(["STANDARD", "ENHANCED"]), + "is_disabled_for_benefits": st.just(True), + "gender": st.sampled_from(["MALE", "FEMALE"]), + # Age 40 on 6 October 2026. + "date_of_birth": st.sampled_from([19860101, 19860601, 19851231]), +} @settings(max_examples=60, deadline=None) @given( - adult_inputs=st.dictionaries( - uk_input_names, st.integers(min_value=1, max_value=10**6), max_size=5 + adult_inputs=st.fixed_dictionaries( + {}, + optional={name: strat for name, strat in UK_PROMPTED_PERSON_INPUTS.items()}, ), - household_inputs=st.dictionaries( - st.sampled_from( - ["rent", "savings", "tenure_type", "mortgage_interest_repayment"] - ), - st.integers(min_value=1, max_value=10**6), - max_size=4, + household_inputs=st.fixed_dictionaries( + {}, + optional={ + "rent": st.integers(min_value=1, max_value=10**5), + "savings": st.integers(min_value=1, max_value=10**6), + "tenure_type": st.sampled_from(["RENT_PRIVATELY", "OWNED_OUTRIGHT"]), + }, ), num_children=st.integers(min_value=0, max_value=3), - wage=st.floats(min_value=0, max_value=10**6, allow_nan=False), + wage=st.integers(min_value=0, max_value=10**6), ) def test_uk_situation_holds_exactly_the_prompted_facts( adult_inputs, household_inputs, num_children, wage ): + from policybench.prompts import describe_household, describe_person + scenario = Scenario( id="uk", country="uk", @@ -1444,7 +1446,12 @@ def test_uk_situation_holds_exactly_the_prompted_facts( Person(name="adult1", age=40, employment_income=wage, inputs=adult_inputs) ], children=[ - Person(name=f"child{i + 1}", age=5, employment_income=0.0) + Person( + name=f"child{i + 1}", + age=17, + employment_income=0.0, + inputs={"current_education": "POST_SECONDARY"}, + ) for i in range(num_children) ], household_inputs=household_inputs, @@ -1452,22 +1459,138 @@ def test_uk_situation_holds_exactly_the_prompted_facts( ) situation = scenario.to_pe_uk_situation(prefix="s__") + prompt = describe_household(scenario) people = situation["people"] assert set(people) == {f"s__{p.name}" for p in scenario.all_people} adult = people["s__adult1"] - assert set(adult) == {"age", "employment_income_before_lsr", *adult_inputs} + structural = {"age", "employment_income_before_lsr", "is_claimant_or_partner"} + assert set(adult) == structural | set(adult_inputs) + assert adult["is_claimant_or_partner"] == {"2026": True} assert adult["employment_income_before_lsr"] == {"2026": wage} + adult_lines = describe_person(scenario.adults[0], country="uk") for key, value in adult_inputs.items(): - expected = int(value) if key == "date_of_birth" else value - assert adult[key] == {"2026": expected} + assert adult[key] == {"2026": value} + # Every input the situation holds is one the prompt states. + if key == "is_disabled_for_benefits": + assert "limited capability for work" in adult_lines + else: + label = key.replace("_", " ") if key != "pip_dl_category" else "PIP" + assert label.split()[0].lower() in adult_lines.lower() + for i in range(num_children): + child = people[f"s__child{i + 1}"] + assert child["is_claimant_or_partner"] == {"2026": False} + assert child["age_started_or_accepted_current_education_or_training"] == { + "2026": scenarios_module.UK_EDUCATION_ENTRY_AGE + } + assert "begun before age 19" in prompt (benunit,) = situation["benunits"].values() (household,) = situation["households"].values() assert benunit["members"] == household["members"] == list(people) + assert benunit["is_married"] == {"2026": False} assert set(household) == {"members", "region", *household_inputs} assert household["region"] == {"2026": "SCOTLAND"} +def test_uk_situation_rejects_inputs_the_prompt_cannot_show(): + base = dict( + id="uk", + country="uk", + state="WALES", + filing_status=None, + year=2026, + ) + hidden = Scenario( + **base, + adults=[ + Person( + name="adult1", + age=70, + employment_income=0.0, + inputs={"state_pension_reported": 9_000.0}, + ) + ], + ) + with pytest.raises(ValueError, match="inputs the prompt does not show"): + hidden.to_pe_uk_situation() + tax_unit = Scenario( + **base, + adults=[Person(name="adult1", age=40, employment_income=0.0)], + tax_unit_inputs={"anything": 1}, + ) + with pytest.raises(ValueError, match="tax-unit or benefit inputs"): + tax_unit.to_pe_uk_situation() + + +def test_uk_private_renters_get_their_region_brma_in_the_prompt(): + from policybench.prompts import describe_household + + row = pd.Series( + { + "household_id": 1, + "household_weight": 1.0, + "region": "SCOTLAND", + "tenure_type": "RENT_PRIVATELY", + "rent": 7_200.4, + } + ) + inputs = scenarios_module._extract_uk_household_inputs(row) + # Region is a scenario field, not a household input. + assert inputs == { + "tenure_type": "RENT_PRIVATELY", + "rent": 7_200.0, + "brma": "GREATER_GLASGOW", + } + owner = scenarios_module._extract_uk_household_inputs( + row.replace("RENT_PRIVATELY", "OWNED_OUTRIGHT") + ) + assert "brma" not in owner + scenario = Scenario( + id="uk", + country="uk", + state="SCOTLAND", + filing_status=None, + adults=[Person(name="adult1", age=40, employment_income=0.0)], + household_inputs=inputs, + year=2026, + ) + assert ( + "Broad Rental Market Area (for the Local Housing Allowance): Greater Glasgow" + in describe_household(scenario) + ) + assert set(scenarios_module.UK_REGION_BRMA) >= { + "LONDON", + "SCOTLAND", + "WALES", + "NORTHERN_IRELAND", + } + + +@pytest.mark.slow +def test_pe_uk_reads_a_prompted_couple_as_an_unmarried_couple(): + """The prompt says two adults are a couple, not married. PE-UK would + otherwise presume a 37/18 pair to be a parent and child, and presume any + couple married (scenario_056 and the Marriage Allowance cells).""" + from policyengine_uk import Simulation + + scenario = Scenario( + id="uk", + country="uk", + state="WALES", + filing_status=None, + adults=[ + Person(name="adult1", age=37, employment_income=30_000.0), + Person(name="adult2", age=18, employment_income=0.0), + ], + household_inputs={"tenure_type": "OWNED_OUTRIGHT"}, + year=2026, + ) + sim = Simulation(situation=scenario.to_pe_uk_situation()) + assert bool(sim.calculate("is_couple", 2026)[0]) + assert not bool(sim.calculate("is_married", 2026)[0]) + assert float(sim.calculate("marriage_allowance", 2026).sum()) == 0 + + def test_uk_situation_rejects_us_scenarios(): scenario = Scenario( id="us", From 5d29ba0d55412eae55c37dfec860bf58b5351f6c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 09:22:57 -0400 Subject: [PATCH 6/9] Give the UK prompt and reference one canonical scenario Address round 2 of the #212 review. The conventions (whole-number amounts, a private renter's Broad Rental Market Area, a non-advanced-education entry age) applied only when scenarios were extracted from the transfer data, so a scenario loaded from a manifest could still give the prompt and the reference different facts. - canonical_uk_scenario applies them, idempotently; describe_household and to_pe_uk_situation both start from it. - The education entry age is a stated fact: an explicit age is kept and shown, 16 only where the scenario gives none (it overwrote explicit ages). - to_pe_uk_situation also rejects household inputs the prompt filters out. - The BRMA mapping is described as a fixed area within each region (Maidstone is not the South East's largest city). - Tests: a loaded unrounded manifest gives prompt and reference the same rounded facts and BRMA; explicit entry ages 18 and 19; hidden household inputs; idempotence (Hypothesis). Co-Authored-By: Claude Opus 5.5 --- docs/benchmark_card.md | 18 ++-- policybench/prompts.py | 10 ++- policybench/scenarios.py | 102 +++++++++++++++++---- tests/test_eval_no_tools.py | 3 +- tests/test_scenarios.py | 172 ++++++++++++++++++++++++++++-------- 5 files changed, 246 insertions(+), 59 deletions(-) diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 408f3744..6123c329 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -436,13 +436,17 @@ so by listing none. Since October 2026 the path works as follows. a couple who are not married or in a civil partnership, and the reference supplies exactly that (claimant and partner flags, no marriage), so neither PE-UK's presumed-child rule nor its presumption that couples are married - applies. A private renter's prompt states a Broad Rental Market Area: the - area of the region's largest city, since the transfer data records none - and PE-UK otherwise reads Maidstone's Local Housing Allowance everywhere. - Someone in non-advanced education is stated to have begun it before 19. -- **Whole-number facts.** Every UK amount is stored as the whole number the - prompt shows, so the reference uses the stated figure, not an unrounded - uprated one. + applies. A private renter's prompt states a Broad Rental Market Area: a + fixed area within each region, since the transfer data records none and + PE-UK otherwise reads Maidstone's Local Housing Allowance everywhere. + Someone in non-advanced education has the age it began stated (16 where + the scenario gives none). +- **Whole-number facts.** Every UK amount is the whole number the prompt + shows, so the reference uses the stated figure, not an unrounded uprated + one. +- **One canonical scenario.** The prompt renderer and the reference builder + both start from `canonical_uk_scenario`, which applies these conventions, + so a scenario loaded from a manifest gives each the same facts. - **Engine-version guards.** Reference generation stops if the artifact stores a variable the installed PolicyEngine UK does not define, or if a derived fact the prompt states is renamed, so a model release cannot silently drop diff --git a/policybench/prompts.py b/policybench/prompts.py index 54d51c90..c16e0a64 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -3,6 +3,7 @@ from policybench.scenarios import ( Person, Scenario, + canonical_uk_scenario, is_excluded_prompt_input_name, ) from policybench.spec import find_output_spec, parse_person_output @@ -160,6 +161,9 @@ "hours_worked": "annual hours worked", "current_education": "current education", "brma": "Broad Rental Market Area (for the Local Housing Allowance)", + "age_started_or_accepted_current_education_or_training": ( + "age when the current education or training began" + ), "date_of_birth": "date of birth", "workers_compensation": "workers' compensation", "capital_gains_before_response": "capital gains", @@ -215,6 +219,7 @@ NON_MONETARY_NUMERIC_FIELDS = { + "age_started_or_accepted_current_education_or_training", "weekly_hours_worked", "hours_worked_last_week", "hours_worked", @@ -264,7 +269,7 @@ def _format_yyyymmdd(value) -> str: "UPPER_SECONDARY": "secondary school", "POST_SECONDARY": ( "full-time non-advanced education (school or college, below " - "higher-education level), begun before age 19" + "higher-education level)" ), "TERTIARY": "advanced (higher) education", } @@ -412,6 +417,9 @@ def _describe_entity_inputs( def describe_household(scenario: Scenario) -> str: """Create a structured description of a household.""" + # The UK reference starts from the same canonical scenario, so the prompt + # and the reference state the same facts however the scenario was built. + scenario = canonical_uk_scenario(scenario) period_label = ( f"UK fiscal year: {scenario.year}-{str(scenario.year + 1)[-2:]}" if scenario.country == "uk" diff --git a/policybench/scenarios.py b/policybench/scenarios.py index e4e8ae5c..d0c16677 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -348,10 +348,10 @@ # Below 16 every person is a child; from 20 no one is a qualifying young person. UK_EDUCATION_PROMPT_AGES = range(16, 20) -# A Broad Rental Market Area per region for private renters: the area of the -# region's largest city. PE-UK reads the Local Housing Allowance from the BRMA -# alone and defaults every household to Maidstone, whatever its region; the -# transfer data records none. The prompt states the area it uses. +# A fixed Broad Rental Market Area within each region, for private renters. +# PE-UK reads the Local Housing Allowance from the BRMA alone and defaults +# every household to Maidstone, whatever its region; the transfer data records +# none. The prompt states the area it uses. UK_REGION_BRMA = { "NORTH_EAST": "TYNESIDE", "NORTH_WEST": "CENTRAL_GREATER_MANCHESTER", @@ -368,11 +368,11 @@ } UK_PRIVATE_RENT_TENURE = "RENT_PRIVATELY" -# The age a prompted person in non-advanced education began it. The data -# records none, and PE-UK's default (1000) would fail every -# qualifying-young-person entry condition; the prompt states "began before -# age 19". +# The age a prompted person in non-advanced education began it, where the +# scenario gives none. PE-UK's default (1000) would fail every +# qualifying-young-person entry condition. The prompt states the age. UK_EDUCATION_ENTRY_AGE = 16 +UK_EDUCATION_ENTRY_FIELD = "age_started_or_accepted_current_education_or_training" UK_NON_PROMPTABLE_SENTINELS = { "", @@ -647,6 +647,15 @@ def to_pe_uk_situation(self, prefix: str = "") -> dict: """ if self.country != "uk": raise ValueError("to_pe_uk_situation is only supported for UK scenarios.") + self = canonical_uk_scenario(self) + hidden_household = sorted( + k for k in self.household_inputs if is_excluded_prompt_input_name(k) + ) + if hidden_household: + raise ValueError( + f"{self.id}: household inputs the prompt does not show: " + f"{hidden_household}" + ) if self.tax_unit_inputs or self.spm_unit_inputs: raise ValueError( f"{self.id}: UK scenarios cannot carry tax-unit or benefit inputs." @@ -674,10 +683,6 @@ def to_pe_uk_situation(self, prefix: str = "") -> dict: if key == "date_of_birth": value = int(value) person_data[key] = self._yearize(value) - if person.inputs.get("current_education") == "POST_SECONDARY": - person_data["age_started_or_accepted_current_education_or_training"] = ( - self._yearize(UK_EDUCATION_ENTRY_AGE) - ) people[f"{prefix}{person.name}"] = person_data members = list(people) household_data = { @@ -751,6 +756,75 @@ def marital_units(self) -> dict: } +def _canonical_uk_inputs(inputs: dict[str, Any]) -> dict[str, Any]: + canonical: dict[str, Any] = {} + for key, value in inputs.items(): + if isinstance(value, (bool, np.bool_, str)) or value is None: + canonical[key] = value + continue + number = round_uk_prompt_number(value) + if number != 0: + canonical[key] = number + return canonical + + +def canonical_uk_scenario(scenario: "Scenario") -> "Scenario": + """A UK scenario holding exactly the facts its prompt states. + + The prompt renderer and the reference builder both start from this, so a + scenario loaded from a manifest gives the same facts to each as one built + from the transfer data: + + - every amount is the whole number the prompt shows, and an amount shown + as 0 is unlisted; + - a private renter without a Broad Rental Market Area gets the fixed one + for the region (``UK_REGION_BRMA``); + - someone in non-advanced education without an entry age gets + ``UK_EDUCATION_ENTRY_AGE``. + + Idempotent. Scenarios of other countries are returned unchanged. + """ + if scenario.country != "uk": + return scenario + + def person(original: Person) -> Person: + inputs = _canonical_uk_inputs(original.inputs) + if inputs.get("current_education") == "POST_SECONDARY": + inputs.setdefault(UK_EDUCATION_ENTRY_FIELD, float(UK_EDUCATION_ENTRY_AGE)) + return Person( + name=original.name, + age=original.age, + employment_income=round_uk_prompt_number(original.employment_income), + inputs=inputs, + ) + + household_inputs = _canonical_uk_inputs(scenario.household_inputs) + if ( + household_inputs.get("tenure_type") == UK_PRIVATE_RENT_TENURE + and "brma" not in household_inputs + ): + if scenario.state not in UK_REGION_BRMA: + raise ValueError( + f"{scenario.id}: no Broad Rental Market Area is set for region " + f"{scenario.state!r}." + ) + household_inputs["brma"] = UK_REGION_BRMA[scenario.state] + return Scenario( + id=scenario.id, + state=scenario.state, + filing_status=scenario.filing_status, + adults=[person(adult) for adult in scenario.adults], + children=[person(child) for child in scenario.children], + tax_unit_inputs=scenario.tax_unit_inputs, + spm_unit_inputs=scenario.spm_unit_inputs, + household_inputs=household_inputs, + year=scenario.year, + country=scenario.country, + source_dataset=scenario.source_dataset, + metadata=scenario.metadata, + ) + + def person_to_dict(person: Person) -> dict[str, Any]: """Serialize a Person to a JSON-safe dict.""" return { @@ -1547,8 +1621,6 @@ def _extract_uk_household_inputs(row: pd.Series) -> dict[str, Any]: promptable = _uk_promptable_value(value) if promptable is not None: inputs[col] = promptable - if inputs.get("tenure_type") == UK_PRIVATE_RENT_TENURE: - inputs["brma"] = UK_REGION_BRMA[str(row["region"])] return inputs @@ -1693,7 +1765,7 @@ def scenarios_from_uk_frames( ) ) - return scenarios + return [canonical_uk_scenario(scenario) for scenario in scenarios] def generate_scenarios( diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index 318b6fd0..d9d5a112 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -4019,8 +4019,9 @@ def test_uk_prompt_states_role_specific_disability_and_birth_facts(): assert "- annual hours worked: 1,820" in prompt assert ( "- current education: full-time non-advanced education (school or " - "college, below higher-education level), begun before age 19" + "college, below higher-education level)" ) in prompt + assert "- age when the current education or training began: 16" in prompt assert "- current education: advanced (higher) education" in prompt diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 6343d8d5..5421d09c 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1480,15 +1480,19 @@ def test_uk_situation_holds_exactly_the_prompted_facts( for i in range(num_children): child = people[f"s__child{i + 1}"] assert child["is_claimant_or_partner"] == {"2026": False} - assert child["age_started_or_accepted_current_education_or_training"] == { + assert child[scenarios_module.UK_EDUCATION_ENTRY_FIELD] == { "2026": scenarios_module.UK_EDUCATION_ENTRY_AGE } - assert "begun before age 19" in prompt + assert "age when the current education or training began: 16" in prompt (benunit,) = situation["benunits"].values() (household,) = situation["households"].values() assert benunit["members"] == household["members"] == list(people) assert benunit["is_married"] == {"2026": False} - assert set(household) == {"members", "region", *household_inputs} + expected_household = {"members", "region", *household_inputs} + if household_inputs.get("tenure_type") == "RENT_PRIVATELY": + expected_household.add("brma") + assert "Broad Rental Market Area" in prompt + assert set(household) == expected_household assert household["region"] == {"2026": "SCOTLAND"} @@ -1522,48 +1526,146 @@ def test_uk_situation_rejects_inputs_the_prompt_cannot_show(): tax_unit.to_pe_uk_situation() -def test_uk_private_renters_get_their_region_brma_in_the_prompt(): - from policybench.prompts import describe_household - - row = pd.Series( - { - "household_id": 1, - "household_weight": 1.0, - "region": "SCOTLAND", - "tenure_type": "RENT_PRIVATELY", - "rent": 7_200.4, - } - ) - inputs = scenarios_module._extract_uk_household_inputs(row) - # Region is a scenario field, not a household input. - assert inputs == { - "tenure_type": "RENT_PRIVATELY", - "rent": 7_200.0, - "brma": "GREATER_GLASGOW", - } - owner = scenarios_module._extract_uk_household_inputs( - row.replace("RENT_PRIVATELY", "OWNED_OUTRIGHT") - ) - assert "brma" not in owner - scenario = Scenario( +def _uk_renter(**overrides) -> Scenario: + fields = dict( id="uk", country="uk", state="SCOTLAND", filing_status=None, - adults=[Person(name="adult1", age=40, employment_income=0.0)], - household_inputs=inputs, + adults=[Person(name="adult1", age=40, employment_income=15_095.1591796875)], + household_inputs={"tenure_type": "RENT_PRIVATELY", "rent": 7_200.4}, year=2026, ) + fields.update(overrides) + return Scenario(**fields) + + +def test_uk_private_renters_get_their_region_brma_in_prompt_and_reference(): + from policybench.prompts import describe_household + + scenario = _uk_renter() + (household,) = scenario.to_pe_uk_situation()["households"].values() + assert household["brma"] == {"2026": "GREATER_GLASGOW"} assert ( "Broad Rental Market Area (for the Local Housing Allowance): Greater Glasgow" in describe_household(scenario) ) - assert set(scenarios_module.UK_REGION_BRMA) >= { - "LONDON", - "SCOTLAND", - "WALES", - "NORTHERN_IRELAND", - } + + stated = _uk_renter( + household_inputs={"tenure_type": "RENT_PRIVATELY", "brma": "LOTHIAN"} + ) + (household,) = stated.to_pe_uk_situation()["households"].values() + assert household["brma"] == {"2026": "LOTHIAN"} + + owner = _uk_renter(household_inputs={"tenure_type": "OWNED_OUTRIGHT"}) + (household,) = owner.to_pe_uk_situation()["households"].values() + assert "brma" not in household + assert "Broad Rental Market Area" not in describe_household(owner) + + with pytest.raises(ValueError, match="no Broad Rental Market Area"): + _uk_renter(state="ATLANTIS").to_pe_uk_situation() + + +def test_a_loaded_uk_manifest_gives_prompt_and_reference_the_same_facts(tmp_path): + from policybench.prompts import describe_household + + # A manifest written before the conventions: unrounded amounts, no BRMA. + manifest = tmp_path / "scenarios.csv" + scenario_manifest([_uk_renter()]).to_csv(manifest, index=False) + (loaded,) = load_scenarios_from_manifest(manifest) + assert loaded.adults[0].employment_income == 15_095.1591796875 + + situation = loaded.to_pe_uk_situation() + (person,) = situation["people"].values() + (household,) = situation["households"].values() + prompt = describe_household(loaded) + assert person["employment_income_before_lsr"] == {"2026": 15_095.0} + assert "gross wages and salaries: £15,095" in prompt + assert household["rent"] == {"2026": 7_200.0} + assert "rent: £7,200" in prompt + assert household["brma"] == {"2026": "GREATER_GLASGOW"} + assert "Greater Glasgow" in prompt + + +@pytest.mark.parametrize("entry_age", [18, 19]) +def test_an_explicit_uk_education_entry_age_is_kept_and_stated(entry_age): + from policybench.prompts import describe_household + + scenario = _uk_renter( + children=[ + Person( + name="child1", + age=19, + employment_income=0.0, + inputs={ + "current_education": "POST_SECONDARY", + scenarios_module.UK_EDUCATION_ENTRY_FIELD: entry_age, + }, + ) + ] + ) + child = scenario.to_pe_uk_situation()["people"]["child1"] + assert child[scenarios_module.UK_EDUCATION_ENTRY_FIELD] == {"2026": entry_age} + assert ( + f"age when the current education or training began: {entry_age}" + in describe_household(scenario) + ) + + +def test_uk_situation_rejects_household_inputs_the_prompt_cannot_show(): + hidden = _uk_renter( + household_inputs={ + "tenure_type": "OWNED_OUTRIGHT", + "bus_fare_spending_reported": 300.0, + } + ) + assert scenarios_module.is_excluded_prompt_input_name("bus_fare_spending_reported") + with pytest.raises(ValueError, match="household inputs the prompt does not show"): + hidden.to_pe_uk_situation() + + +uk_numbers = st.floats(min_value=-1e6, max_value=1e6, allow_nan=False) + + +@settings(max_examples=100, deadline=None) +@given( + wage=uk_numbers, + savings=uk_numbers, + rent=uk_numbers, + tenure=st.sampled_from(["RENT_PRIVATELY", "OWNED_OUTRIGHT"]), + child_in_education=st.booleans(), +) +def test_canonical_uk_scenario_is_idempotent( + wage, savings, rent, tenure, child_in_education +): + scenario = _uk_renter( + adults=[ + Person( + name="adult1", + age=40, + employment_income=wage, + inputs={"savings_interest_income": savings, "gender": "FEMALE"}, + ) + ], + children=[ + Person( + name="child1", + age=17, + employment_income=0.0, + inputs={"current_education": "POST_SECONDARY"} + if child_in_education + else {}, + ) + ], + household_inputs={"tenure_type": tenure, "rent": rent}, + ) + once = scenarios_module.canonical_uk_scenario(scenario) + twice = scenarios_module.canonical_uk_scenario(once) + assert scenarios_module.scenario_to_dict(once) == scenarios_module.scenario_to_dict( + twice + ) + # The original is untouched. + assert scenario.adults[0].employment_income == wage @pytest.mark.slow From 652b679f483424b2e509b7325e307380c5b08a22 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 09:52:37 -0400 Subject: [PATCH 7/9] Canonicalise UK people by field; state gainful self-employment and relief method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address round 3 of the #212 review and two conventions the reference audit surfaced. - Explicit zeros are stated facts and are kept (a zero months_since_last_birthday moved the date of birth when dropped); rate fields keep their precision; months render without a currency sign. - canonical_uk_person, shared by describe_person and the reference: someone aged 16 to 19 with no education status is "not in education" (PE-UK imputed enrolment from age), and someone with self-employment income and no gainful self-employment determination has none (PE-UK presumed one and applied the Universal Credit minimum income floor). Both follow the prompt's rule that an unlisted status is false, and both are stated. - Pension contribution labels state the relief method: employee contributions under a net pay arrangement, personal contributions as relief at source. On the frozen run this moves three Universal Credit references; two land on the model consensus and the audit judge's own answer (scenario_044 £8,442.06, scenario_047 £15,873.48). Co-Authored-By: Claude Opus 5.5 --- policybench/prompts.py | 23 +++++--- policybench/scenarios.py | 69 ++++++++++++++++-------- tests/test_scenarios.py | 111 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 174 insertions(+), 29 deletions(-) diff --git a/policybench/prompts.py b/policybench/prompts.py index c16e0a64..33cd7df8 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -1,8 +1,10 @@ """Prompt templates for PolicyBench evaluations.""" from policybench.scenarios import ( + RATE_OR_RATIO_FIELD_SUFFIXES, Person, Scenario, + canonical_uk_person, canonical_uk_scenario, is_excluded_prompt_input_name, ) @@ -88,7 +90,17 @@ "early_withdrawal_penalty": "early withdrawal penalty", "educator_expense": "educator expense", "employment_income": "gross wages and salaries", - "employee_pension_contributions_reported": "employee pension contributions", + "employee_pension_contributions_reported": ( + "employee pension contributions (net pay arrangement: taken from pay " + "before tax)" + ), + "personal_pension_contributions": ( + "personal pension contributions (relief at source)" + ), + "uc_is_in_gainful_self_employment": ( + "determined to be in gainful self-employment for Universal Credit" + ), + "months_since_last_birthday": "months since last birthday", "estate_income": "estate income", "excess_withheld_payroll_tax": "excess withheld payroll tax", "farm_income": "farm income", @@ -219,6 +231,7 @@ NON_MONETARY_NUMERIC_FIELDS = { + "months_since_last_birthday", "age_started_or_accepted_current_education_or_training", "weekly_hours_worked", "hours_worked_last_week", @@ -274,11 +287,6 @@ def _format_yyyymmdd(value) -> str: "TERTIARY": "advanced (higher) education", } -RATE_OR_RATIO_FIELD_SUFFIXES = ( - "_rate", - "_ratio", -) - def _currency_symbol(country: str) -> str: return "£" if country == "uk" else "$" @@ -375,6 +383,9 @@ def _person_heading(person: Person, country: str = "us") -> str: def describe_person(person: Person, country: str = "us") -> str: """Create a structured description of a person.""" + if country == "uk": + # The same canonical person the UK reference is built from. + person = canonical_uk_person(person) lines = [ f"{_person_heading(person, country=country)}:", f"- age: {person.age}", diff --git a/policybench/scenarios.py b/policybench/scenarios.py index d0c16677..a6611429 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -756,18 +756,56 @@ def marital_units(self) -> dict: } +# Fields the prompt shows as a fraction rather than a whole number. The +# renderer formats by the same suffixes. +RATE_OR_RATIO_FIELD_SUFFIXES = ("_rate", "_ratio") +UK_GAINFUL_SELF_EMPLOYMENT_FIELD = "uc_is_in_gainful_self_employment" + + def _canonical_uk_inputs(inputs: dict[str, Any]) -> dict[str, Any]: + """Each numeric input as the prompt shows it. An explicit zero is a stated + fact and is kept; rates keep their precision.""" canonical: dict[str, Any] = {} for key, value in inputs.items(): if isinstance(value, (bool, np.bool_, str)) or value is None: canonical[key] = value - continue - number = round_uk_prompt_number(value) - if number != 0: - canonical[key] = number + elif key.endswith(RATE_OR_RATIO_FIELD_SUFFIXES): + canonical[key] = float(value) + else: + canonical[key] = round_uk_prompt_number(value) return canonical +def canonical_uk_person(person: Person) -> Person: + """A UK person holding exactly the facts the prompt states about them. + + Besides whole-number amounts, three statuses PE-UK would otherwise fill in + itself are made explicit, each following the prompt's rule that an + unlisted status is false: + + - someone aged 16 to 19 with no education status is not in education + (PE-UK imputes enrolment from age); + - someone in non-advanced education with no entry age began it at + ``UK_EDUCATION_ENTRY_AGE``; + - someone with self-employment income and no gainful self-employment + determination has none (PE-UK presumes one, applying the Universal + Credit minimum income floor). + """ + inputs = _canonical_uk_inputs(person.inputs) + if int(person.age) in UK_EDUCATION_PROMPT_AGES: + inputs.setdefault("current_education", "NOT_IN_EDUCATION") + if inputs.get("current_education") == "POST_SECONDARY": + inputs.setdefault(UK_EDUCATION_ENTRY_FIELD, float(UK_EDUCATION_ENTRY_AGE)) + if inputs.get("self_employment_income"): + inputs.setdefault(UK_GAINFUL_SELF_EMPLOYMENT_FIELD, False) + return Person( + name=person.name, + age=person.age, + employment_income=round_uk_prompt_number(person.employment_income), + inputs=inputs, + ) + + def canonical_uk_scenario(scenario: "Scenario") -> "Scenario": """A UK scenario holding exactly the facts its prompt states. @@ -775,29 +813,16 @@ def canonical_uk_scenario(scenario: "Scenario") -> "Scenario": scenario loaded from a manifest gives the same facts to each as one built from the transfer data: - - every amount is the whole number the prompt shows, and an amount shown - as 0 is unlisted; + - every amount is the whole number the prompt shows; + - each person is canonical (:func:`canonical_uk_person`); - a private renter without a Broad Rental Market Area gets the fixed one - for the region (``UK_REGION_BRMA``); - - someone in non-advanced education without an entry age gets - ``UK_EDUCATION_ENTRY_AGE``. + for the region (``UK_REGION_BRMA``). Idempotent. Scenarios of other countries are returned unchanged. """ if scenario.country != "uk": return scenario - def person(original: Person) -> Person: - inputs = _canonical_uk_inputs(original.inputs) - if inputs.get("current_education") == "POST_SECONDARY": - inputs.setdefault(UK_EDUCATION_ENTRY_FIELD, float(UK_EDUCATION_ENTRY_AGE)) - return Person( - name=original.name, - age=original.age, - employment_income=round_uk_prompt_number(original.employment_income), - inputs=inputs, - ) - household_inputs = _canonical_uk_inputs(scenario.household_inputs) if ( household_inputs.get("tenure_type") == UK_PRIVATE_RENT_TENURE @@ -813,8 +838,8 @@ def person(original: Person) -> Person: id=scenario.id, state=scenario.state, filing_status=scenario.filing_status, - adults=[person(adult) for adult in scenario.adults], - children=[person(child) for child in scenario.children], + adults=[canonical_uk_person(adult) for adult in scenario.adults], + children=[canonical_uk_person(child) for child in scenario.children], tax_unit_inputs=scenario.tax_unit_inputs, spm_unit_inputs=scenario.spm_unit_inputs, household_inputs=household_inputs, diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 5421d09c..20e75d82 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1716,7 +1716,8 @@ def test_uk_rounding_keeps_the_displayed_number(value): rounded = scenarios_module.round_uk_prompt_number(value) assert rounded == int(rounded) if rounded == 0: - # Shown as £0 (or £-0) before; now unlisted, which the prompt reads as 0. + # Extraction leaves an amount shown as £0 (or £-0) unlisted, which the + # prompt reads as 0. assert scenarios_module._uk_promptable_value(value) is None return # The prompt shows the rounded value exactly as it showed the raw one, so @@ -1731,3 +1732,111 @@ def test_uk_promptable_values_are_whole_and_drop_amounts_shown_as_zero(): assert scenarios_module._uk_promptable_value(-107_891.4) == -107_891.0 assert scenarios_module._uk_promptable_value(0.4) is None assert scenarios_module._uk_promptable_value(0.6) == 1.0 + + +def test_canonical_uk_person_keeps_explicit_zeros_and_rates(): + from policybench.prompts import describe_person + + person = Person( + name="adult1", + age=66, + employment_income=0.0, + inputs={ + "months_since_last_birthday": 0, + "full_rate_vat_expenditure_rate": 0.25, + "savings_interest_income": 0.4, + }, + ) + canonical = scenarios_module.canonical_uk_person(person) + # An explicit zero is a stated fact: dropping it would let PE-UK use its + # own default (six months), moving the date of birth. + assert canonical.inputs["months_since_last_birthday"] == 0.0 + assert canonical.inputs["full_rate_vat_expenditure_rate"] == 0.25 + assert canonical.inputs["savings_interest_income"] == 0.0 + lines = describe_person(person, country="uk") + assert "- months since last birthday: 0" in lines + assert "- savings interest income: £0" in lines + + +def test_canonical_uk_person_states_statuses_pe_uk_would_impute(): + from policybench.prompts import describe_person + + teenager = Person(name="child1", age=17, employment_income=0.0) + canonical = scenarios_module.canonical_uk_person(teenager) + # PE-UK imputes non-advanced education at 17; the prompt's rule is that an + # unlisted status is false. + assert canonical.inputs["current_education"] == "NOT_IN_EDUCATION" + assert scenarios_module.UK_EDUCATION_ENTRY_FIELD not in canonical.inputs + assert "- current education: not in education or training" in describe_person( + teenager, country="uk" + ) + assert ( + "current_education" + not in scenarios_module.canonical_uk_person( + Person(name="child1", age=12, employment_income=0.0) + ).inputs + ) + + trader = Person( + name="adult1", + age=40, + employment_income=0.0, + inputs={"self_employment_income": 3_900.0}, + ) + canonical = scenarios_module.canonical_uk_person(trader) + assert canonical.inputs[scenarios_module.UK_GAINFUL_SELF_EMPLOYMENT_FIELD] is False + assert ( + "- determined to be in gainful self-employment for Universal Credit: no" + in describe_person(trader, country="uk") + ) + stated = scenarios_module.canonical_uk_person( + Person( + name="adult1", + age=40, + employment_income=0.0, + inputs={ + "self_employment_income": 3_900.0, + scenarios_module.UK_GAINFUL_SELF_EMPLOYMENT_FIELD: True, + }, + ) + ) + assert stated.inputs[scenarios_module.UK_GAINFUL_SELF_EMPLOYMENT_FIELD] is True + + +def test_describe_person_and_describe_household_agree_for_uk(): + from policybench.prompts import describe_household, describe_person + + scenario = _uk_renter( + adults=[ + Person( + name="adult1", + age=40, + employment_income=15_095.16, + inputs={"self_employment_income": 3_900.4}, + ) + ], + children=[Person(name="child1", age=17, employment_income=0.0)], + ) + household = describe_household(scenario) + for person in scenario.all_people: + assert describe_person(person, country="uk") in household + + +@pytest.mark.slow +def test_pe_uk_applies_no_minimum_income_floor_without_a_determination(): + from policyengine_uk import Simulation + + scenario = _uk_renter( + adults=[ + Person( + name="adult1", + age=40, + employment_income=0.0, + inputs={"self_employment_income": 3_900.0}, + ) + ], + household_inputs={"tenure_type": "OWNED_OUTRIGHT"}, + ) + sim = Simulation(situation=scenario.to_pe_uk_situation()) + assert not bool(sim.calculate("uc_mif_applies", 2026).any()) + assert float(sim.calculate("universal_credit", 2026).sum()) > 0 From 25ba522ce2cf044f2dbdc5baf3ae86a8c1e85773 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 16:55:31 -0400 Subject: [PATCH 8/9] State the pension convention the UK reference follows; store rates as shown Round-4 review of the UK revival. - The prompt called personal pension contributions "relief at source", but PolicyEngine UK deducts them from taxable income, so the label described a method the reference does not follow. It now states the convention: a gross amount deducted from taxable income. - A UK rate reached the engine at full precision while the prompt showed four significant figures. The canonical scenario now stores the rate the prompt shows. Co-Authored-By: Claude Opus 5.5 --- docs/benchmark_card.md | 12 +++++++++--- policybench/prompts.py | 7 ++++++- policybench/scenarios.py | 16 +++++++++++++--- tests/test_scenarios.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 7 deletions(-) diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 6123c329..4c27ba72 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -441,9 +441,15 @@ so by listing none. Since October 2026 the path works as follows. PE-UK otherwise reads Maidstone's Local Housing Allowance everywhere. Someone in non-advanced education has the age it began stated (16 where the scenario gives none). -- **Whole-number facts.** Every UK amount is the whole number the prompt - shows, so the reference uses the stated figure, not an unrounded uprated - one. +- **Facts as shown.** Every UK amount is the whole number the prompt shows, + and every rate the four significant figures it shows, so the reference uses + the stated figure, not an unrounded uprated one. +- **Pension contributions.** PolicyEngine UK deducts both kinds of + contribution from taxable income. The prompt says so for each: employee + contributions under a net pay arrangement, and personal contributions as a + gross amount deducted from taxable income by benchmark convention. It does + not call personal contributions relief at source, which works differently + and which the reference does not model. - **One canonical scenario.** The prompt renderer and the reference builder both start from `canonical_uk_scenario`, which applies these conventions, so a scenario loaded from a manifest gives each the same facts. diff --git a/policybench/prompts.py b/policybench/prompts.py index 33cd7df8..bd5c19d3 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -94,8 +94,13 @@ "employee pension contributions (net pay arrangement: taken from pay " "before tax)" ), + # PE-UK deducts personal contributions from taxable income, as it does + # net pay contributions. Relief at source works differently (the provider + # claims basic rate relief), so the label states the convention the + # reference follows rather than a method it does not. "personal_pension_contributions": ( - "personal pension contributions (relief at source)" + "personal pension contributions (gross amount; benchmark convention: " + "deducted from taxable income)" ), "uc_is_in_gainful_self_employment": ( "determined to be in gainful self-employment for Universal Credit" diff --git a/policybench/scenarios.py b/policybench/scenarios.py index a6611429..9e280649 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -763,14 +763,15 @@ def marital_units(self) -> dict: def _canonical_uk_inputs(inputs: dict[str, Any]) -> dict[str, Any]: - """Each numeric input as the prompt shows it. An explicit zero is a stated - fact and is kept; rates keep their precision.""" + """Each numeric input as the prompt shows it: amounts to the whole unit, + rates to four significant figures. An explicit zero is a stated fact and + is kept.""" canonical: dict[str, Any] = {} for key, value in inputs.items(): if isinstance(value, (bool, np.bool_, str)) or value is None: canonical[key] = value elif key.endswith(RATE_OR_RATIO_FIELD_SUFFIXES): - canonical[key] = float(value) + canonical[key] = round_uk_prompt_rate(value) else: canonical[key] = round_uk_prompt_number(value) return canonical @@ -1594,6 +1595,15 @@ def round_uk_prompt_number(value: float) -> float: return float(round(float(value))) +def round_uk_prompt_rate(value: float) -> float: + """A UK rate or ratio as the prompt shows it: to four significant figures. + + The prompt formats rates with ``.4g``, so the stored value is the one that + format prints and the reference uses exactly the stated rate. + """ + return float(f"{float(value):.4g}") + + def _uk_promptable_value(value: Any) -> Any | None: if pd.isna(value): return None diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 20e75d82..2d574494 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1758,6 +1758,34 @@ def test_canonical_uk_person_keeps_explicit_zeros_and_rates(): assert "- savings interest income: £0" in lines +def test_canonical_uk_rate_is_the_rate_the_prompt_shows(): + from policybench.prompts import describe_household + + scenario = _uk_renter( + household_inputs={ + "tenure_type": "OWNED_OUTRIGHT", + "full_rate_vat_expenditure_rate": 0.123456789, + } + ) + canonical = scenarios_module.canonical_uk_scenario(scenario) + assert canonical.household_inputs["full_rate_vat_expenditure_rate"] == 0.1235 + assert "full-rate VAT expenditure share: 0.1235" in describe_household(scenario) + situation = scenario.to_pe_uk_situation() + (household,) = situation["households"].values() + assert household["full_rate_vat_expenditure_rate"] == {"2026": 0.1235} + + +@settings(max_examples=200, deadline=None) +@given(rate=st.floats(min_value=0, max_value=10, allow_nan=False, allow_infinity=False)) +def test_uk_prompt_rate_round_trips_through_its_display(rate): + stored = scenarios_module.round_uk_prompt_rate(rate) + # The prompt's format prints the stored value exactly, and storing is + # idempotent, so the reference and the prompt hold the same rate. + assert float(f"{stored:,.4g}".replace(",", "")) == stored + assert f"{stored:,.4g}" == f"{rate:,.4g}" + assert scenarios_module.round_uk_prompt_rate(stored) == stored + + def test_canonical_uk_person_states_statuses_pe_uk_would_impute(): from policybench.prompts import describe_person From 869e9bb284e75234376c6f2137fb79168598b789 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 18:46:49 -0400 Subject: [PATCH 9/9] State that UK pension contributions do not reduce adjusted net income Round-5 review of the UK revival. PolicyEngine UK deducts both kinds of pension contribution from taxable income but neither from adjusted net income, so they do not lower the High Income Child Benefit Charge or the personal allowance taper (policyengine-uk #2243). Both prompt labels and the benchmark card now say so. The rate round-trip property also covers negative rates and values around the thousands separator. Co-Authored-By: Claude Opus 5.5 --- docs/benchmark_card.md | 12 +++++++----- policybench/prompts.py | 15 +++++++++------ tests/test_scenarios.py | 18 ++++++++++++++++-- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 4c27ba72..fd409a42 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -445,11 +445,13 @@ so by listing none. Since October 2026 the path works as follows. and every rate the four significant figures it shows, so the reference uses the stated figure, not an unrounded uprated one. - **Pension contributions.** PolicyEngine UK deducts both kinds of - contribution from taxable income. The prompt says so for each: employee - contributions under a net pay arrangement, and personal contributions as a - gross amount deducted from taxable income by benchmark convention. It does - not call personal contributions relief at source, which works differently - and which the reference does not model. + contribution from taxable income and neither from adjusted net income, so + they do not lower the High Income Child Benefit Charge or the personal + allowance taper (policyengine-uk #2243). The prompt states that convention + for each: employee contributions under a net pay arrangement, and personal + contributions as a gross amount. It does not call personal contributions + relief at source, which works differently and which the reference does not + model. - **One canonical scenario.** The prompt renderer and the reference builder both start from `canonical_uk_scenario`, which applies these conventions, so a scenario loaded from a manifest gives each the same facts. diff --git a/policybench/prompts.py b/policybench/prompts.py index bd5c19d3..e81afed3 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -90,17 +90,20 @@ "early_withdrawal_penalty": "early withdrawal penalty", "educator_expense": "educator expense", "employment_income": "gross wages and salaries", + # PE-UK deducts both kinds of contribution from taxable income and + # neither from adjusted net income, which sets the High Income Child + # Benefit Charge and the personal allowance taper (policyengine-uk + # #2243). Relief at source works differently (the provider claims basic + # rate relief), so the labels state the convention the reference + # follows rather than a method it does not. "employee_pension_contributions_reported": ( "employee pension contributions (net pay arrangement: taken from pay " - "before tax)" + "before income tax; benchmark convention: not deducted from adjusted " + "net income)" ), - # PE-UK deducts personal contributions from taxable income, as it does - # net pay contributions. Relief at source works differently (the provider - # claims basic rate relief), so the label states the convention the - # reference follows rather than a method it does not. "personal_pension_contributions": ( "personal pension contributions (gross amount; benchmark convention: " - "deducted from taxable income)" + "deducted from taxable income, not from adjusted net income)" ), "uc_is_in_gainful_self_employment": ( "determined to be in gainful self-employment for Universal Credit" diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 2d574494..70826026 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1775,8 +1775,22 @@ def test_canonical_uk_rate_is_the_rate_the_prompt_shows(): assert household["full_rate_vat_expenditure_rate"] == {"2026": 0.1235} -@settings(max_examples=200, deadline=None) -@given(rate=st.floats(min_value=0, max_value=10, allow_nan=False, allow_infinity=False)) +@settings(max_examples=300, deadline=None) +@given( + rate=st.one_of( + st.floats(min_value=-10, max_value=10, allow_nan=False, allow_infinity=False), + # Around the thousands separator and .4g's switch to exponent form. + st.floats( + min_value=-200_000, + max_value=200_000, + allow_nan=False, + allow_infinity=False, + ), + st.sampled_from( + [999.95, 1_000.0, 1_234.5, -1_234.5, 9_999.4, 9_999.6, 12_345.0] + ), + ) +) def test_uk_prompt_rate_round_trips_through_its_display(rate): stored = scenarios_module.round_uk_prompt_rate(rate) # The prompt's format prints the stored value exactly, and storing is