From d1063ff679b6f668fb0c45de4ed475b8737173df Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:09:07 -0400 Subject: [PATCH 1/8] Add a stratified UK sample and audit outputs for round 2 of the policyengine-uk audit Round 1 of the internal UK audit drew 100 unstratified households from the public transfer dataset, so pensioners, carers, childcare, Scotland and council tax were barely exercised. This adds what round 2 needs: - policybench/uk_audit.py: a seeded, stratified 200-household sample (12 strata) that selects transfer-data households by structure and states the facts the data lacks (State Pension, Attendance Allowance, caring hours, childcare costs, social tenancies, a council and its Council Tax liability), each from a documented distribution. - policybench/uk_audit_councils.csv: 47 councils with their 2026-27 Band D and the page each figure was read from, and the rental area a private renter there is placed in. - A uk_audit output spec: the seven UK headline outputs plus the High Income Child Benefit Charge, Carer's Allowance, Attendance Allowance, Housing Benefit, Council Tax Reduction, Tax-Free Childcare and Scottish Child Payment. The UK headline set is unchanged. - Prompts state the new facts in words; State Pension becomes promptable. - `policybench uk-audit-sample` and `policybench run --program-set`. - policyengine-uk 2.125.1 -> 2.126.7 (US pins unchanged). Co-Authored-By: Claude Opus 5.5 --- policybench/benchmark_specs.json | 161 ++++++ policybench/cli.py | 28 + policybench/prompts.py | 53 +- policybench/scenarios.py | 5 + policybench/supervisor.py | 15 +- policybench/uk_audit.py | 883 ++++++++++++++++++++++++++++++ policybench/uk_audit_councils.csv | 48 ++ pyproject.toml | 4 +- tests/test_supervisor.py | 35 ++ tests/test_uk_audit.py | 373 +++++++++++++ uv.lock | 8 +- 11 files changed, 1603 insertions(+), 10 deletions(-) create mode 100644 policybench/uk_audit.py create mode 100644 policybench/uk_audit_councils.csv create mode 100644 tests/test_uk_audit.py diff --git a/policybench/benchmark_specs.json b/policybench/benchmark_specs.json index dce2f24c..f7279220 100644 --- a/policybench/benchmark_specs.json +++ b/policybench/benchmark_specs.json @@ -311,6 +311,167 @@ } ] } + }, + "uk_audit": { + "description": "Internal policyengine-uk audit (round 2, October 2026): the UK headline outputs plus the High Income Child Benefit Charge, Carer's Allowance, Attendance Allowance, Housing Benefit, Council Tax Reduction, Tax-Free Childcare and Scottish Child Payment. Not a public board.", + "countries": { + "uk": [ + { + "id": "income_tax", + "pe_variable": "income_tax", + "label": "Income Tax", + "prompt": "household total annual UK Income Tax liability after allowances and reliefs, excluding Capital Gains Tax", + "metric_type": "amount", + "role": "tax", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": -1 + }, + { + "id": "national_insurance", + "pe_variable": "national_insurance", + "label": "National Insurance", + "prompt": "household total annual UK National Insurance contributions, excluding employer National Insurance", + "metric_type": "amount", + "role": "tax", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": -1 + }, + { + "id": "capital_gains_tax", + "pe_variable": "capital_gains_tax", + "label": "Capital Gains Tax", + "prompt": "household total annual UK Capital Gains Tax. Compute it separately from Income Tax. UK has no short-term versus long-term capital gains distinction; treat any listed capital gains as ordinary net chargeable gains before the annual exempt amount, excluding carried interest and special reliefs unless explicitly stated. Sum across household members", + "metric_type": "amount", + "role": "tax", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": -1 + }, + { + "id": "child_benefit", + "pe_variable": "child_benefit", + "label": "Child Benefit", + "prompt": "household total annual gross Child Benefit amount for qualifying children and young people before the High Income Child Benefit Charge; assume eligible households take up the benefit and do not require stated benefit receipt; do not apply an income test or tax-charge reduction to this output; do not subtract HICBC here because it is included in Income Tax, so report gross Child Benefit even when HICBC would recover it through tax", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "universal_credit", + "pe_variable": "universal_credit", + "label": "Universal Credit", + "prompt": "household total annual Universal Credit amount; assume eligible households take up the benefit and do not require stated benefit receipt", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "pension_credit", + "pe_variable": "pension_credit", + "label": "Pension Credit", + "prompt": "annual Pension Credit amount", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "pip", + "pe_variable": "pip", + "label": "Personal Independence Payment", + "prompt": "annual Personal Independence Payment (PIP) amount", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "high_income_child_benefit_charge", + "pe_variable": "CB_HITC", + "label": "High Income Child Benefit Charge", + "prompt": "household total annual High Income Child Benefit Charge on the Child Benefit the household receives (0 if none). The Income Tax output includes this charge; report the charge here on its own as well", + "metric_type": "amount", + "role": "tax", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": -1 + }, + { + "id": "carers_allowance", + "pe_variable": "carers_allowance", + "label": "Carer's Allowance", + "prompt": "household total annual Carer's Allowance payable, after any reduction for an overlapping benefit such as the State Pension; assume anyone entitled claims it", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "attendance_allowance", + "pe_variable": "attendance_allowance", + "label": "Attendance Allowance", + "prompt": "household total annual Attendance Allowance", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "housing_benefit", + "pe_variable": "housing_benefit", + "label": "Housing Benefit", + "prompt": "household total annual Housing Benefit; assume anyone entitled claims it", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "council_tax_reduction", + "pe_variable": "council_tax_reduction", + "label": "Council Tax Reduction", + "prompt": "household total annual Council Tax Reduction (council tax support) under the scheme of the household's council, as a reduction in the stated Council Tax bill; assume anyone entitled claims it", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "tax_free_childcare", + "pe_variable": "tax_free_childcare", + "label": "Tax-Free Childcare", + "prompt": "household total annual government top-up paid through Tax-Free Childcare for the stated childcare costs; assume the household claims every benefit it is entitled to, including Universal Credit, and that an eligible family opens an account and pays all the stated childcare costs through it", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + }, + { + "id": "scottish_child_payment", + "pe_variable": "scottish_child_payment", + "label": "Scottish Child Payment", + "prompt": "household total annual Scottish Child Payment; assume anyone entitled claims it", + "metric_type": "amount", + "role": "benefit", + "output_set": "headline", + "aggregation": "sum", + "net_income_sign": 1 + } + ] + } } } } diff --git a/policybench/cli.py b/policybench/cli.py index 0adfeeff..efef6a43 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -1019,6 +1019,11 @@ def main(): help="Stop dispatching at 90%% of this spend; omit for no cap", ) run_parser.add_argument("--max-workers", type=int, default=4) + run_parser.add_argument( + "--program-set", + default=None, + help="Benchmark output set the workers request (default: headline)", + ) run_parser.add_argument( "--max-rounds", type=int, @@ -1039,6 +1044,16 @@ def main(): "--report-output", default=None, help="Optional path for the report" ) + uk_audit_parser = subparsers.add_parser( + "uk-audit-sample", + help="Draw the stratified UK household sample for the policyengine-uk " + "audit and write its scenario manifest", + ) + uk_audit_parser.add_argument( + "-o", "--output", default="results/local/uk_audit/scenarios.csv" + ) + uk_audit_parser.add_argument("--seed", type=int, default=None) + fold_parser = subparsers.add_parser( "fold-board", help="Fold new model prediction CSVs into an existing board and " @@ -1090,6 +1105,18 @@ def main(): enable_cache() + if args.command == "uk-audit-sample": + import json + + from policybench.uk_audit import UK_AUDIT_SEED, write_uk_audit_sample + + meta = write_uk_audit_sample( + args.output, seed=UK_AUDIT_SEED if args.seed is None else args.seed + ) + print(json.dumps(meta["counts"], indent=1)) + print(f"UK audit sample saved to {args.output}") + return + if args.command == "reference-outputs": from policybench.ground_truth import calculate_ground_truth from policybench.policyengine_runtime import runtime_metadata_for_country @@ -1197,6 +1224,7 @@ def main(): budget_usd=args.budget_usd, max_workers=args.max_workers, max_rounds=args.max_rounds, + program_set=args.program_set, ) state = supervisor.run() except ValueError as exc: diff --git a/policybench/prompts.py b/policybench/prompts.py index e81afed3..5b183730 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -1,5 +1,7 @@ """Prompt templates for PolicyBench evaluations.""" +from functools import lru_cache + from policybench.scenarios import ( RATE_OR_RATIO_FIELD_SUFFIXES, Person, @@ -188,8 +190,19 @@ "workers_compensation": "workers' compensation", "capital_gains_before_response": "capital gains", "communication_consumption": "communication spending", - "council_tax": "Council Tax", + "council_tax": ( + "annual Council Tax liability (after the single person discount, if " + "any; before Council Tax Reduction)" + ), "council_tax_band": "Council Tax band", + "local_authority": "council (local authority)", + "aa_category": "Attendance Allowance award", + "care_hours": "hours a week spent caring for the other adult in the household", + "childcare_expenses": "annual childcare costs", + "tax_free_childcare_uses_qualifying_provider": ( + "childcare provider is registered or approved (qualifies for " + "Tax-Free Childcare)" + ), "corporate_wealth": "corporate financial wealth", "dividend_income": "dividend income", "education_consumption": "education spending", @@ -230,7 +243,7 @@ "savings": "savings", "savings_interest_income": "savings interest income", "state_pension": "State Pension income", - "state_pension_reported": "reported State Pension income", + "state_pension_reported": "State Pension (amount for 2026-27)", "tenure_type": "tenure", "transport_consumption": "transport spending", "selected_marketplace_plan_benchmark_ratio": ("selected Marketplace plan premium"), @@ -239,6 +252,7 @@ NON_MONETARY_NUMERIC_FIELDS = { + "care_hours", "months_since_last_birthday", "age_started_or_accepted_current_education_or_training", "weekly_hours_worked", @@ -296,6 +310,38 @@ def _format_yyyymmdd(value) -> str: } +# UK category values in words. Others print title-cased. +UK_VALUE_DESCRIPTIONS = { + ("tenure_type", "OWNED_OUTRIGHT"): "owned outright", + ("tenure_type", "OWNED_WITH_MORTGAGE"): "owned with a mortgage", + ("tenure_type", "RENT_PRIVATELY"): "renting privately", + ("tenure_type", "RENT_FROM_COUNCIL"): "renting from the council", + ("tenure_type", "RENT_FROM_HA"): "renting from a housing association", + ("aa_category", "LOWER"): "lower rate", + ("aa_category", "HIGHER"): "higher rate", +} + + +@lru_cache(maxsize=1) +def _uk_local_authority_names() -> dict[str, str]: + """PE-UK's names for councils ("Bristol, City of"), keyed by enum name.""" + try: + from policyengine_uk.variables.household.demographic.locations import ( + LocalAuthority, + ) + except ImportError: + return {} + return {member.name: member.value for member in LocalAuthority} + + +def _uk_value_description(field: str, value: str) -> str | None: + if (field, value) in UK_VALUE_DESCRIPTIONS: + return UK_VALUE_DESCRIPTIONS[(field, value)] + if field == "local_authority": + return _uk_local_authority_names().get(value) + return None + + def _currency_symbol(country: str) -> str: return "£" if country == "uk" else "$" @@ -375,6 +421,9 @@ def _format_input_line(field: str, value, country: str = "us") -> str: if field == "current_education" and value in UK_EDUCATION_DESCRIPTIONS: return f"- {label}: {UK_EDUCATION_DESCRIPTIONS[value]}" if isinstance(value, str): + described = _uk_value_description(field, value) if country == "uk" else None + if described is not None: + return f"- {label}: {described}" return f"- {label}: {value.replace('_', ' ').title()}" if field == "selected_marketplace_plan_benchmark_ratio": return _format_marketplace_plan_ratio(float(value)) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 9e280649..31dbf412 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -196,6 +196,11 @@ PROMPTABLE_REPORTED_INPUTS = { "employee_pension_contributions_reported", + # A household simulated on its own has no dataset year, so PE-UK reads + # this as the State Pension for the period itself (new_state_pension, + # basic_state_pension and additional_state_pension split it at that + # year's rates). The prompt states it as the 2026-27 amount. + "state_pension_reported", } INPUT_NAME_ALIASES = { diff --git a/policybench/supervisor.py b/policybench/supervisor.py index 51358d23..5420d627 100644 --- a/policybench/supervisor.py +++ b/policybench/supervisor.py @@ -139,8 +139,12 @@ def __init__( max_rounds: int = 4, python: str | None = None, env: dict | None = None, + program_set: str | None = None, ): self.model = model + # None keeps the configured default set, so existing runs' workload + # fingerprints and worker commands are unchanged. + self.program_set = program_set self.litellm_id = MODELS.get(model, model) from policybench.eval_no_tools import _reject_sensitivity_knobs_for_responses @@ -189,7 +193,12 @@ def _load_scenarios(self): def _programs_for(self, index: int) -> list[str]: """The output set the worker CLI requests for one scenario's country.""" - return get_programs(self.scenarios[index].country) + return self._programs_for_country(self.scenarios[index].country) + + def _programs_for_country(self, country: str) -> list[str]: + if self.program_set is None: + return get_programs(country) + return get_programs(country, self.program_set) def _load_initial_request_variables(self) -> list[str]: """Expand the first manifest row exactly as the worker CLI will.""" @@ -210,7 +219,7 @@ def _workload(self) -> dict: { program for scenario in self.scenarios - for program in get_programs(scenario.country) + for program in self._programs_for_country(scenario.country) } ) ), @@ -689,6 +698,8 @@ def _spawn(self, index: int) -> subprocess.Popen: "-o", str(out), ] + if self.program_set is not None: + cmd.extend(["--program-set", self.program_set]) return subprocess.Popen( cmd, stdout=open(log, "w"), diff --git a/policybench/uk_audit.py b/policybench/uk_audit.py new file mode 100644 index 00000000..cc29c702 --- /dev/null +++ b/policybench/uk_audit.py @@ -0,0 +1,883 @@ +"""A stratified UK household sample for auditing policyengine-uk. + +Round 1 of the UK audit (October 2026) drew 100 households from the public UK +transfer dataset the way the US leg draws from its own data: weighted, without +strata. Most of them were working-age owner-occupiers or private renters in +England, so whole areas of the UK rules were barely touched. The transfer +dataset also carries no State Pension, no caring hours, no childcare costs, +no Attendance Allowance and no social tenancies. + +This module draws round 2's households by stratum. Each stratum selects +transfer-data households with the structure it needs (a region, an age band, +children, a tenure) and then states the facts the data lacks: a State +Pension amount, caring hours, childcare costs, a council and its council tax +bill, a rental area. Every such fact is drawn from a documented distribution +with a seeded generator and stated in the prompt, and the reference reads +exactly those facts (``Scenario.to_pe_uk_situation``). The drawn values are +constructed, not observed: they are plausible circumstances chosen to +exercise the rules, not estimates of any population. + +The sample is for finding engine defects. It is not representative of the +UK and is never weighted to the population. +""" + +from __future__ import annotations + +import csv +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd + +from policybench.config import TAX_YEAR +from policybench.scenarios import ( + UK_PRIVATE_RENT_TENURE, + Person, + Scenario, + _build_uk_person, + _eligible_uk_households, + _extract_uk_household_inputs, + canonical_uk_scenario, +) + +UK_AUDIT_SEED = 20261011 +UK_AUDIT_SOURCE = "uk_calibrated_transfer_stratified_audit" + +# State Pension age in 2026-27. The Pensions Act 2014 raises it from 66 to 67 +# for people born on or after 6 April 1960, so a 66-year-old may not yet have +# reached it; the reference reads the stated date of birth. Strata that need +# pensioners select adults aged 67 and over. +PENSIONER_MIN_AGE = 67 +WORKING_AGE_MAX = 64 + +# Weekly State Pension rates for 2026-27 (policyengine-uk 2.126.7 parameters +# gov.dwp.state_pension.*.amount at 2026-06-01). Used only to construct +# plausible State Pension amounts; the reference reads the stated amount. +NEW_STATE_PENSION_WEEKLY = 241.30 +BASIC_STATE_PENSION_WEEKLY = 184.90 +# Pensions Act 2014 s. 4: the new State Pension applies to people who reach +# pensionable age on or after 6 April 2016, so everyone born before 6 April +# 1951 is on the basic scheme. +NEW_STATE_PENSION_FIRST_BIRTH_DATE = 19510406 +WEEKS_PER_YEAR = 52 + +# Statutory council tax band ratios to Band D (Local Government Finance Act +# 1992 s. 5(1) for England, as amended for Wales by the Council Tax (Valuation +# Bands) (Wales) Order 2003; Scotland: Council Tax (Substitution of +# Proportion) (Scotland) Order 2016). +ENGLAND_BAND_RATIOS = { + "A": 6 / 9, + "B": 7 / 9, + "C": 8 / 9, + "D": 1, + "E": 11 / 9, + "F": 13 / 9, + "G": 15 / 9, + "H": 18 / 9, +} +WALES_BAND_RATIOS = {**ENGLAND_BAND_RATIOS, "I": 21 / 9} +SCOTLAND_BAND_RATIOS = { + "A": 240 / 360, + "B": 280 / 360, + "C": 320 / 360, + "D": 1, + "E": 473 / 360, + "F": 585 / 360, + "G": 705 / 360, + "H": 882 / 360, +} + + +@dataclass(frozen=True) +class Council: + """A council the sample can place a household in. + + ``band_d`` is the council's 2026-27 Band D council tax, used only to + state a liability; ``band_d_source`` is where it was read. ``brma`` is + the Broad Rental Market Area a private renter there is placed in. + """ + + local_authority: str + region: str + band_d: float + brma: str + band_d_source: str = "" + + +COUNCILS_PATH = Path(__file__).with_name("uk_audit_councils.csv") + +# Working-age English councils whose Council Tax Reduction schemes +# policyengine-uk simulates (council_tax_reduction/config.py). Elsewhere in +# England a working-age claim falls back to a reported reduction +# (council_tax_reduction_scheme_supported), so its reference is not a +# function of the prompt. +SIMULATED_WORKING_AGE_CTR_COUNCILS = ( + "MERTON", + "KINGSTON_UPON_THAMES", + "NEWHAM", + "WESTMINSTER", + "OXFORD", +) + +# The single person discount (Local Government Finance Act 1992 s. 11(1)): +# a quarter off where one adult is resident. +SINGLE_PERSON_DISCOUNT = 0.25 + + +def load_councils(path: Path = COUNCILS_PATH) -> dict[str, tuple[Council, ...]]: + """Councils by region, from the packaged table. Each Band D figure was + read from the council's or government's published 2026-27 figures; the + table cites the page. Councils whose figure could not be read from an + official page are left out.""" + grouped: dict[str, list[Council]] = {} + with path.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle): + council = Council( + local_authority=row["local_authority"], + region=row["region"], + band_d=float(row["band_d"]), + brma=row["brma"], + band_d_source=row.get("band_d_source", ""), + ) + grouped.setdefault(council.region, []).append(council) + return {region: tuple(items) for region, items in grouped.items()} + + +COUNCILS: dict[str, tuple[Council, ...]] = load_councils() + + +def band_ratio(region: str, band: str) -> float: + if region == "SCOTLAND": + return SCOTLAND_BAND_RATIOS[band] + if region == "WALES": + return WALES_BAND_RATIOS[band] + return ENGLAND_BAND_RATIOS[band] + + +def valid_band(region: str, band: str) -> str: + """The transfer data's band if the nation has it; Band I exists only in + Wales, so elsewhere it becomes H.""" + table = ( + SCOTLAND_BAND_RATIOS + if region == "SCOTLAND" + else WALES_BAND_RATIOS + if region == "WALES" + else ENGLAND_BAND_RATIOS + ) + if band in table: + return band + return "H" + + +@dataclass(frozen=True) +class Household: + """A transfer-data household with the features strata select on.""" + + household_id: int + region: str + tenure: str + weight: float + adult_ages: tuple[int, ...] + child_ages: tuple[int, ...] + adult_earnings: tuple[float, ...] + adult_self_employment: tuple[float, ...] + adult_income: tuple[float, ...] + savings: float + + +def household_features( + person_df: pd.DataFrame, household_df: pd.DataFrame +) -> list[Household]: + """Features of every household the benchmark can prompt (one benefit + unit, one or two adults).""" + eligible = _eligible_uk_households(person_df, household_df) + people = person_df[person_df["person_household_id"].isin(eligible["household_id"])] + income_columns = [ + column + for column in ( + "employment_income_before_lsr", + "self_employment_income", + "private_pension_income", + "property_income", + "savings_interest_income", + "dividend_income", + "miscellaneous_income", + ) + if column in people.columns + ] + features = [] + by_household = dict(tuple(people.groupby("person_household_id"))) + for row in eligible.itertuples(index=False): + members = by_household[int(row.household_id)] + adults = members[members["age"] >= 18] + children = members[members["age"] < 18] + features.append( + Household( + household_id=int(row.household_id), + region=str(row.region), + tenure=str(row.tenure_type), + weight=float(row.household_weight), + adult_ages=tuple(int(a) for a in adults["age"]), + child_ages=tuple(int(a) for a in children["age"]), + adult_earnings=tuple( + float(v) for v in adults["employment_income_before_lsr"] + ), + adult_self_employment=tuple( + float(v) for v in adults["self_employment_income"] + ), + adult_income=tuple( + float(v) for v in adults[income_columns].sum(axis=1) + ), + savings=float(getattr(row, "savings", 0.0) or 0.0), + ) + ) + return features + + +Selector = Callable[[Household], bool] +Overlay = Callable[[Scenario, Household, np.random.Generator], Scenario] + + +@dataclass(frozen=True) +class Stratum: + name: str + count: int + description: str + select: Selector + overlay: Overlay + + +# --- selectors ----------------------------------------------------------- + + +def _all_working_age(h: Household) -> bool: + return all(age <= WORKING_AGE_MAX for age in h.adult_ages) + + +def _gb(h: Household) -> bool: + return h.region != "NORTHERN_IRELAND" + + +def select_scotland(h: Household) -> bool: + return h.region == "SCOTLAND" + + +# Low income: every adult's transfer-data income (before any State Pension) +# below £15,000 and household savings below £16,000 (the Universal Credit and +# Housing Benefit capital limit), so the means-tested benefits have cells to +# test. +LOW_INCOME = 15_000 +LOW_SAVINGS = 16_000 + + +def _low_income(h: Household) -> bool: + return max(h.adult_income) < LOW_INCOME and h.savings < LOW_SAVINGS + + +def select_scotland_low_income_families(h: Household) -> bool: + return ( + h.region == "SCOTLAND" + and any(age < 16 for age in h.child_ages) + and _low_income(h) + and _all_working_age(h) + ) + + +def select_pension_age_low_income(h: Household) -> bool: + return select_pension_age(h) and _low_income(h) + + +def select_private_renters_low_income(h: Household) -> bool: + return select_private_renters(h) and _low_income(h) + + +def select_pension_age(h: Household) -> bool: + return ( + _gb(h) + and all(age >= PENSIONER_MIN_AGE for age in h.adult_ages) + and not h.child_ages + ) + + +def select_carers(h: Household) -> bool: + return ( + _gb(h) + and h.region != "SCOTLAND" + and len(h.adult_ages) == 2 + and _all_working_age(h) + and all(age >= 18 for age in h.adult_ages) + ) + + +def select_childcare(h: Household) -> bool: + has_young_child = any(age <= 11 for age in h.child_ages) + every_adult_works = all( + earnings > 0 or self_employment > 0 + for earnings, self_employment in zip(h.adult_earnings, h.adult_self_employment) + ) + return ( + _gb(h) + and h.region != "SCOTLAND" + and has_young_child + and every_adult_works + and _all_working_age(h) + ) + + +def select_child_benefit_charge(h: Household) -> bool: + top = max(h.adult_income) + return ( + _gb(h) + and h.region != "SCOTLAND" + and bool(h.child_ages) + and 55_000 <= top <= 90_000 + and _all_working_age(h) + ) + + +def select_allowance_taper(h: Household) -> bool: + top = max(h.adult_income) + return ( + _gb(h) + and h.region != "SCOTLAND" + and 95_000 <= top <= 135_000 + and _all_working_age(h) + ) + + +def select_self_employed_loss(h: Household) -> bool: + return ( + _gb(h) + and h.region != "SCOTLAND" + and any(value < 0 for value in h.adult_self_employment) + and _all_working_age(h) + ) + + +def select_private_renters(h: Household) -> bool: + return ( + _gb(h) + and h.region != "SCOTLAND" + and h.tenure == UK_PRIVATE_RENT_TENURE + and _all_working_age(h) + ) + + +def select_any(h: Household) -> bool: + """Any Great Britain household. Northern Ireland has domestic rates, not + Council Tax, so the sample leaves it out.""" + return _gb(h) + + +# --- overlays ------------------------------------------------------------ + + +def _with_person_inputs(person: Person, **inputs: Any) -> Person: + merged = dict(person.inputs) + for key, value in inputs.items(): + if value is None: + merged.pop(key, None) + else: + merged[key] = value + return replace(person, inputs=merged) + + +def _drop_gift_aid(scenario: Scenario) -> Scenario: + """Gift Aid is set to nil outside the control stratum: policyengine-uk + gives it by deduction rather than by extending the rate bands (issue + #2253, fix #2271 not yet released), which round 1 already measured.""" + return replace( + scenario, + adults=[_with_person_inputs(a, gift_aid=None) for a in scenario.adults], + children=[_with_person_inputs(c, gift_aid=None) for c in scenario.children], + ) + + +def _council_for( + scenario: Scenario, + rng: np.random.Generator, + prefer_simulated_ctr: bool, +) -> Council: + councils = COUNCILS.get(scenario.state) + if not councils: + raise ValueError(f"No councils are set for region {scenario.state!r}.") + if prefer_simulated_ctr: + simulated = [ + c + for c in councils + if c.local_authority in SIMULATED_WORKING_AGE_CTR_COUNCILS + ] + if simulated: + councils = tuple(simulated) + return councils[int(rng.integers(len(councils)))] + + +def state_council( + scenario: Scenario, + rng: np.random.Generator, + band: str, + prefer_simulated_ctr: bool = False, +) -> Scenario: + """Place the household in a council and state its Council Tax liability. + + The liability is the council's 2026-27 Band D times the statutory ratio + for the household's band, less the single person discount where one + adult lives there. policyengine-uk reads ``council_tax`` as the liability + before Council Tax Reduction, so the prompt states it after the discount + and says so. A private renter is placed in the council's rental area. + """ + council = _council_for(scenario, rng, prefer_simulated_ctr) + band = valid_band(scenario.state, band) + inputs = dict(scenario.household_inputs) + inputs["local_authority"] = council.local_authority + inputs["council_tax_band"] = band + charge = council.band_d * band_ratio(scenario.state, band) + if len(scenario.adults) == 1: + charge *= 1 - SINGLE_PERSON_DISCOUNT + inputs["council_tax"] = float(round(charge)) + if inputs.get("tenure_type") == UK_PRIVATE_RENT_TENURE: + inputs["brma"] = council.brma + return replace(scenario, household_inputs=inputs) + + +def _date_of_birth(person: Person) -> int | None: + value = person.inputs.get("date_of_birth") + return None if value is None else int(value) + + +def draw_state_pension(person: Person, rng: np.random.Generator) -> float: + """An annual 2026-27 State Pension for someone over pensionable age. + + New State Pension (pensionable age reached on or after 6 April 2016): + 70% the full rate, 20% a partial rate of 60-99% of it (a shorter + National Insurance record), 10% the full rate plus a protected payment + of £5-£60 a week. Basic State Pension: 25% the full basic rate, 60% the + full basic rate plus £10-£120 a week of additional State Pension, 15% a + partial basic rate of 60-99%. + """ + dob = _date_of_birth(person) + on_new_scheme = dob is not None and dob >= NEW_STATE_PENSION_FIRST_BIRTH_DATE + u = rng.random() + if on_new_scheme: + if u < 0.70: + weekly = NEW_STATE_PENSION_WEEKLY + elif u < 0.90: + weekly = NEW_STATE_PENSION_WEEKLY * rng.uniform(0.60, 0.99) + else: + weekly = NEW_STATE_PENSION_WEEKLY + rng.uniform(5, 60) + else: + if u < 0.25: + weekly = BASIC_STATE_PENSION_WEEKLY + elif u < 0.85: + weekly = BASIC_STATE_PENSION_WEEKLY + rng.uniform(10, 120) + else: + weekly = BASIC_STATE_PENSION_WEEKLY * rng.uniform(0.60, 0.99) + return float(round(weekly * WEEKS_PER_YEAR)) + + +# Social rents for pension-age social tenants: £85-£150 a week. +SOCIAL_RENT_WEEKLY = (85, 150) +SOCIAL_TENURES = ("RENT_FROM_COUNCIL", "RENT_FROM_HA") + + +def state_pensions(scenario: Scenario, rng: np.random.Generator) -> Scenario: + """A State Pension (``draw_state_pension``) for every adult aged 67 or + over who has none stated. Everyone 67 or over in 2026-27 has reached + pensionable age.""" + adults = [ + _with_person_inputs( + adult, state_pension_reported=draw_state_pension(adult, rng) + ) + if adult.age >= PENSIONER_MIN_AGE + and not adult.inputs.get("state_pension_reported") + else adult + for adult in scenario.adults + ] + return replace(scenario, adults=adults) + + +def overlay_pension_age( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + """Attendance Allowance for 30% of adults without PIP daily living (20% + the lower rate, 10% the higher); 35% of owner-occupiers become + social tenants paying £85-£150 a week, since the transfer data has + none. The State Pension comes from ``state_pensions``.""" + adults = [] + for adult in scenario.adults: + attendance = None + if adult.inputs.get("pip_dl_category") in (None, "NONE"): + u = rng.random() + attendance = "LOWER" if u < 0.20 else "HIGHER" if u < 0.30 else None + adults.append(_with_person_inputs(adult, aa_category=attendance)) + inputs = dict(scenario.household_inputs) + if inputs.get("tenure_type") != UK_PRIVATE_RENT_TENURE and rng.random() < 0.35: + inputs["tenure_type"] = SOCIAL_TENURES[int(rng.integers(2))] + inputs["rent"] = float(round(rng.uniform(*SOCIAL_RENT_WEEKLY) * WEEKS_PER_YEAR)) + scenario = replace(scenario, adults=adults, household_inputs=inputs) + return state_council(scenario, rng, band=_band(scenario, rng)) + + +# Caring hours: a fifth below the 35-hour threshold. +CARE_HOURS = (20, 35, 40, 50, 60, 80) +CARE_HOURS_P = (0.15, 0.20, 0.20, 0.20, 0.13, 0.12) + + +def overlay_carers( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + """The lower earner cares for the other adult. + + Caring hours are drawn from ``CARE_HOURS``. In 80% of households the + cared-for adult has the PIP daily living component (standard or + enhanced); in 20% they have no disability benefit. In 60% the carer's + pay is replaced by £0-£300 a week, spanning the Carer's Allowance + earnings limit; otherwise the transfer data's pay stands. + """ + carer_index = int(np.argmin([a.employment_income for a in scenario.adults])) + cared_index = 1 - carer_index + adults = list(scenario.adults) + carer = adults[carer_index] + hours = float(rng.choice(CARE_HOURS, p=CARE_HOURS_P)) + if rng.random() < 0.60: + weekly_pay = float(rng.uniform(0, 300)) + carer = replace( + carer, employment_income=float(round(weekly_pay * WEEKS_PER_YEAR)) + ) + adults[carer_index] = _with_person_inputs(carer, care_hours=hours) + cared = adults[cared_index] + if rng.random() < 0.80: + component = "STANDARD" if rng.random() < 0.5 else "ENHANCED" + adults[cared_index] = _with_person_inputs(cared, pip_dl_category=component) + else: + adults[cared_index] = _with_person_inputs(cared, pip_dl_category=None) + scenario = replace(scenario, adults=adults) + return state_council( + scenario, rng, band=_band(scenario, rng), prefer_simulated_ctr=True + ) + + +def overlay_childcare( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + """Childcare costs for every child aged 11 or under, paid to a + registered provider: £3,000-£15,000 a year under 5, £500-£5,000 from 5 + to 11.""" + children = [] + for child in scenario.children: + if child.age <= 4: + cost = rng.uniform(3_000, 15_000) + elif child.age <= 11: + cost = rng.uniform(500, 5_000) + else: + children.append(child) + continue + children.append( + _with_person_inputs( + child, + childcare_expenses=float(round(cost)), + tax_free_childcare_uses_qualifying_provider=True, + ) + ) + scenario = replace(scenario, children=children) + return state_council( + scenario, rng, band=_band(scenario, rng), prefer_simulated_ctr=True + ) + + +def overlay_self_employed_loss( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + """Half the loss-making self-employed are in gainful self-employment + for Universal Credit; half are not.""" + adults = [] + for adult in scenario.adults: + if adult.inputs.get("self_employment_income", 0) < 0: + adult = _with_person_inputs( + adult, uc_is_in_gainful_self_employment=bool(rng.random() < 0.5) + ) + adults.append(adult) + scenario = replace(scenario, adults=adults) + return state_council( + scenario, rng, band=_band(scenario, rng), prefer_simulated_ctr=True + ) + + +def overlay_council_only( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + return state_council( + scenario, rng, band=_band(scenario, rng), prefer_simulated_ctr=True + ) + + +def overlay_private_renters( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + """Any council in the region, so renters spread across rental areas.""" + return state_council(scenario, rng, band=_band(scenario, rng)) + + +def overlay_scotland( + scenario: Scenario, household: Household, rng: np.random.Generator +) -> Scenario: + return state_council(scenario, rng, band=_band(scenario, rng)) + + +def _band(scenario: Scenario, rng: np.random.Generator) -> str: + band = scenario.metadata.get("council_tax_band") + if band: + return str(band) + return str(rng.choice(list("ABCDE"))) + + +UK_AUDIT_STRATA: tuple[Stratum, ...] = ( + Stratum( + "scotland_low_income_families", + 10, + "Scotland: a child under 16, every adult's income under £15,000 and " + "savings under £16,000", + select_scotland_low_income_families, + overlay_scotland, + ), + Stratum( + "scotland", + 20, + "Scottish taxpayers: transfer-data households in Scotland", + select_scotland, + overlay_scotland, + ), + Stratum( + "pension_age_low_income", + 20, + "Every adult 67 or over, no children, every adult's income other than " + "the State Pension under £15,000, savings under £16,000", + select_pension_age_low_income, + overlay_pension_age, + ), + Stratum( + "pension_age", + 20, + "Every adult 67 or over, no children; State Pension and Attendance " + "Allowance stated; some social tenants", + select_pension_age, + overlay_pension_age, + ), + Stratum( + "carers", + 25, + "Two working-age adults in England or Wales; one cares for the other", + select_carers, + overlay_carers, + ), + Stratum( + "childcare", + 25, + "Every adult working, a child aged 11 or under; childcare costs stated", + select_childcare, + overlay_childcare, + ), + Stratum( + "child_benefit_charge", + 15, + "Children, and an adult with income of £55,000-£90,000", + select_child_benefit_charge, + overlay_council_only, + ), + Stratum( + "allowance_taper", + 10, + "An adult with income of £95,000-£135,000", + select_allowance_taper, + overlay_council_only, + ), + Stratum( + "self_employed_loss", + 20, + "A working-age adult with a self-employment loss", + select_self_employed_loss, + overlay_self_employed_loss, + ), + Stratum( + "private_renters_low_income", + 10, + "Working-age private renters, every adult's income under £15,000 and " + "savings under £16,000", + select_private_renters_low_income, + overlay_private_renters, + ), + Stratum( + "private_renters", + 10, + "Working-age private renters across rental areas", + select_private_renters, + overlay_private_renters, + ), + Stratum( + "control", + 15, + "Unstratified draw in Great Britain, as in round 1 (Gift Aid kept)", + select_any, + overlay_council_only, + ), +) + + +def _scenario_for( + household_id: int, + person_df: pd.DataFrame, + household_lookup: pd.DataFrame, + index: int, + year: int, + dataset_year: int | None, + stratum: str, +) -> Scenario: + household_row = household_lookup.loc[household_id] + members = person_df[person_df["person_household_id"] == household_id].sort_values( + by=["age", "person_id"], ascending=[False, True] + ) + adults, children = [], [] + for _, row in members.iterrows(): + if int(row["age"]) >= 18: + adults.append(_build_uk_person(row, f"adult{len(adults) + 1}")) + else: + children.append(_build_uk_person(row, f"child{len(children) + 1}")) + metadata: dict[str, Any] = { + "household_id": int(household_id), + "stratum": stratum, + "benunit_ids": sorted( + {int(b) for b in members["person_benunit_id"].dropna().astype(int)} + ), + } + band = household_row.get("council_tax_band") + if isinstance(band, str) and band: + metadata["council_tax_band"] = band + if dataset_year is not None: + metadata["dataset_year"] = int(dataset_year) + household_inputs = _extract_uk_household_inputs(household_row) + # The transfer data's council tax comes from US property tax and its + # bands from US home values; the sample states its own bill instead. + household_inputs.pop("council_tax", None) + household_inputs.pop("council_tax_band", None) + return Scenario( + id=f"scenario_{index:03d}", + country="uk", + state=str(household_row["region"]), + filing_status=None, + adults=adults, + children=children, + household_inputs=household_inputs, + year=year, + source_dataset=UK_AUDIT_SOURCE, + metadata=metadata, + ) + + +def _weighted_draw( + candidates: list[Household], n: int, rng: np.random.Generator, name: str = "" +) -> list[Household]: + if len(candidates) < n: + raise ValueError( + f"Stratum {name!r} has {len(candidates)} candidate households for " + f"{n} draws." + ) + weights = np.array([max(c.weight, 0.0) for c in candidates], dtype=float) + p = weights / weights.sum() if weights.sum() > 0 else None + picks = rng.choice(len(candidates), size=n, replace=False, p=p) + return [candidates[int(i)] for i in picks] + + +def stratified_uk_audit_scenarios( + person_df: pd.DataFrame, + household_df: pd.DataFrame, + strata: Sequence[Stratum] = UK_AUDIT_STRATA, + seed: int = UK_AUDIT_SEED, + year: int = TAX_YEAR, + dataset_year: int | None = None, + excluded_household_ids: set[int] | None = None, +) -> list[Scenario]: + """Draw each stratum in order, without replacement across strata. + + Within a stratum, households are drawn with probability proportional to + their transfer-data weight. Each stratum has its own generator, spawned + from ``seed``, so changing one stratum's count or overlay leaves the + draws of the strata before it unchanged. + """ + features = household_features(person_df, household_df) + if excluded_household_ids: + features = [f for f in features if f.household_id not in excluded_household_ids] + household_lookup = household_df.set_index("household_id") + taken: set[int] = set() + scenarios: list[Scenario] = [] + generators = np.random.default_rng(seed).spawn(len(strata)) + for stratum, rng in zip(strata, generators): + candidates = [ + f for f in features if f.household_id not in taken and stratum.select(f) + ] + for household in _weighted_draw(candidates, stratum.count, rng, stratum.name): + taken.add(household.household_id) + scenario = _scenario_for( + household.household_id, + person_df, + household_lookup, + index=len(scenarios), + year=year, + dataset_year=dataset_year, + stratum=stratum.name, + ) + if stratum.name != "control": + scenario = _drop_gift_aid(scenario) + scenario = state_pensions(scenario, rng) + scenario = stratum.overlay(scenario, household, rng) + scenarios.append(canonical_uk_scenario(scenario)) + return scenarios + + +def stratum_counts(scenarios: Sequence[Scenario]) -> Mapping[str, int]: + counts: dict[str, int] = {} + for scenario in scenarios: + name = str(scenario.metadata.get("stratum")) + counts[name] = counts.get(name, 0) + 1 + return counts + + +def write_uk_audit_sample(output: str | Path, seed: int = UK_AUDIT_SEED) -> dict: + """Draw the sample from the pinned transfer dataset and write its + scenario manifest, with a sidecar recording what it was drawn from.""" + import json + from importlib.metadata import version + + from policybench.scenarios import ( + _hash_file, + get_uk_dataset_path, + load_uk_transfer_frames, + scenario_manifest, + ) + + output = Path(output) + person_df, household_df, dataset_year = load_uk_transfer_frames() + scenarios = stratified_uk_audit_scenarios( + person_df, household_df, seed=seed, dataset_year=dataset_year + ) + output.parent.mkdir(parents=True, exist_ok=True) + scenario_manifest(scenarios).to_csv(output, index=False) + dataset = get_uk_dataset_path() + meta = { + "seed": seed, + "strata": { + s.name: {"count": s.count, "description": s.description} + for s in UK_AUDIT_STRATA + }, + "counts": dict(stratum_counts(scenarios)), + "dataset": dataset.name, + "dataset_sha256": _hash_file(dataset), + "dataset_year": dataset_year, + "councils_sha256": _hash_file(COUNCILS_PATH), + "policyengine_uk": version("policyengine-uk"), + "policyengine_core": version("policyengine-core"), + } + Path(f"{output}.meta.json").write_text(json.dumps(meta, indent=1) + "\n") + return meta diff --git a/policybench/uk_audit_councils.csv b/policybench/uk_audit_councils.csv new file mode 100644 index 00000000..29c30e9b --- /dev/null +++ b/policybench/uk_audit_councils.csv @@ -0,0 +1,48 @@ +local_authority,region,brma,band_d,band_d_source +MERTON,LONDON,OUTER_SOUTH_WEST_LONDON,2146.76,https://www.merton.gov.uk/council-tax-benefits-and-housing/council-tax/about/guide +KINGSTON_UPON_THAMES,LONDON,OUTER_SOUTH_WEST_LONDON,2608.12,"https://www.kingston.gov.uk/council-tax/bands-and-charges (standard charge, excluding the Commons levy)" +NEWHAM,LONDON,INNER_EAST_LONDON,1944.23,https://www.newham.gov.uk/council-tax/council-tax-bands-charges-appeals/1 +WESTMINSTER,LONDON,CENTRAL_LONDON,1047.85,https://www.westminster.gov.uk/sites/default/files/media/documents/your-guide-to-council-tax-2026-27.pdf (standard charge) +CROYDON,LONDON,OUTER_SOUTH_LONDON,2599.91,https://www.croydon.gov.uk/council-tax/what-council-tax-and-how-much-it/council-tax-bands +CAMDEN,LONDON,CENTRAL_LONDON,2207.12,https://www.camden.gov.uk/council-tax-guide (standard charge) +OXFORD,SOUTH_EAST,OXFORD,2678.4,https://mycouncil.oxford.gov.uk/ieDecisionDetails.aspx?Id=2829 +READING,SOUTH_EAST,READING,2612.77,https://www.reading.gov.uk/council-tax/council-tax-bands/ +MAIDSTONE,SOUTH_EAST,MAIDSTONE,2454.24,https://maidstone.gov.uk/home/primary-services/council-tax/council-tax-summary +NEWCASTLE_UPON_TYNE,NORTH_EAST,TYNESIDE,2540.66,https://new.newcastle.gov.uk/council-tax/council-tax-explained/council-tax-rates-all-years +SUNDERLAND,NORTH_EAST,SUNDERLAND,2196.25,https://www.sunderland.gov.uk/article/36111/What-is-Council-Tax +MANCHESTER,NORTH_WEST,CENTRAL_GREATER_MANCHESTER,2312.04,https://www.manchester.gov.uk/council-tax/how-council-tax-is-worked-out-data-and-policies/council-tax-bands-and-charges +LIVERPOOL,NORTH_WEST,GREATER_LIVERPOOL,2673.59,https://liverpool.gov.uk/council-tax/how-much/ +BOLTON,NORTH_WEST,BOLTON_AND_BURY,2392.33,https://www.bolton.gov.uk/downloads/file/1945/council-tax-bill-band-d-2026-27 +LANCASTER,NORTH_WEST,LANCASTER,2461.08,https://www.lancaster.gov.uk/council-tax/annual-council-tax-charge +LEEDS,YORKSHIRE,LEEDS,2271.51,https://www.leeds.gov.uk/council-tax/bands-and-charges +SHEFFIELD,YORKSHIRE,SHEFFIELD,2505.32,https://www.sheffield.gov.uk/council-tax/your-bill/check-council-tax-band +BRADFORD,YORKSHIRE,BRADFORD_SOUTH_DALES,2333.05,https://www.bradford.gov.uk/council-tax/bands-and-charges/council-tax-bands-and-amounts/ +YORK,YORKSHIRE,YORK,2269.91,https://www.york.gov.uk/CouncilTaxCharges +NOTTINGHAM,EAST_MIDLANDS,NOTTINGHAM,2755.39,https://www.nottinghamcity.gov.uk/ctbands +LEICESTER,EAST_MIDLANDS,LEICESTER,2528.75,https://www.leicester.gov.uk/council-tax/council-tax-bands-and-charges +DERBY,EAST_MIDLANDS,DERBY,2306.0,https://www.derby.gov.uk/advice-and-benefits/council-tax/bands-charges-and-premiums/ +LINCOLN,EAST_MIDLANDS,LINCOLN,2323.08,https://www.lincoln.gov.uk/council-tax/bands-charges +BIRMINGHAM,WEST_MIDLANDS,BIRMINGHAM,2353.17,Birmingham City Council 2026-27 budget report (birmingham.cmis.uk.com) +COVENTRY,WEST_MIDLANDS,COVENTRY,2516.09,https://www.coventry.gov.uk/council-tax/council-tax-bands-charges +WOLVERHAMPTON,WEST_MIDLANDS,BLACK_COUNTRY,2538.99,https://www.wolverhampton.gov.uk/council-tax/bands-charges +HEREFORDSHIRE_COUNTY_OF,WEST_MIDLANDS,HEREFORDSHIRE,2574.37,https://www.herefordshire.gov.uk/council-tax/what-your-council-tax-pays-for/how-your-council-tax-bill-is-calculated/ +CAMBRIDGE,EAST_OF_ENGLAND,CAMBRIDGE,2467.02,https://www.cambridge.gov.uk/council-tax-bands-and-charges +NORWICH,EAST_OF_ENGLAND,CENTRAL_NORFOLK_NORWICH,2503.43,https://www.norwich.gov.uk/council-tax/understanding-council-tax +LUTON,EAST_OF_ENGLAND,LUTON,2439.93,https://www.luton.gov.uk/sites/default/files/2026-03/council-tax-booklet.pdf +COLCHESTER,EAST_OF_ENGLAND,COLCHESTER,2240.94,https://www.new.colchester.gov.uk/council-tax/council-tax-information-0/council-tax-bands-and-charges +BRISTOL_CITY_OF,SOUTH_WEST,BRISTOL,2713.68,https://www.bristol.gov.uk/residents/council-tax/charges-and-bands +PLYMOUTH,SOUTH_WEST,PLYMOUTH,2441.85,https://www.plymouth.gov.uk/council-tax-bands-and-charges +EXETER,SOUTH_WEST,EXETER,2495.36,https://committees.exeter.gov.uk/documents/g7857/Public%20minutes%2025th-Feb-2026%2018.00%20Council.pdf +CORNWALL,SOUTH_WEST,KERNOW_WEST,2590.93,https://www.cornwall.gov.uk/council-tax/your-council-tax-bill/council-tax-2026/council-tax-bands-2026/ +CARDIFF,WALES,CARDIFF,2008.86,https://cardiff.moderngov.co.uk/mgAi.aspx?ID=38608 +SWANSEA,WALES,SWANSEA,2212.52,https://swansea.moderngov.co.uk/documents/s116741/Minutes%20-%202026Mar05.pdf +NEWPORT,WALES,NEWPORT,2079.59,https://www.newport.gov.uk/sites/default/files/media/media-documents/Council-tax-and-non-domestic-rates-2026-27.pdf +RHONDDA_CYNON_TAF,WALES,TAFF_RHONDDA,2250.41,https://www.rctcbc.gov.uk/EN/Resident/CouncilTax/Suggested/CouncilTaxBandsandCharges/Counciltaxbandsandcharges.aspx +GLASGOW_CITY,SCOTLAND,GREATER_GLASGOW,1706.0,Glasgow City Council revenue estimates 2026-27 +CITY_OF_EDINBURGH,SCOTLAND,LOTHIAN,1626.05,https://www.edinburgh.gov.uk/downloads/file/39461/council-finance-information-2026 +FIFE,SCOTLAND,FIFE,1573.69,https://www.fife.gov.uk/housing/council-tax/how-your-council-tax-is-spent +ABERDEEN_CITY,SCOTLAND,ABERDEEN_AND_SHIRE,1747.54,Aberdeen City Council decisions 4 March 2026 +HIGHLAND,SCOTLAND,HIGHLAND_AND_ISLANDS,1633.99,https://www.highland.gov.uk/council-tax/council-tax-scottish-water-charges/2 +DUNDEE_CITY,SCOTLAND,DUNDEE_AND_ANGUS,1729.69,https://dundeecity.gov.uk/news/article?article_ref=5057 +NORTH_LANARKSHIRE,SCOTLAND,NORTH_LANARKSHIRE,1554.56,https://www.northlanarkshire.gov.uk/council-tax/council-tax-bands-charges-and-appeals +SOUTH_LANARKSHIRE,SCOTLAND,SOUTH_LANARKSHIRE,1468.47,https://www.southlanarkshire.gov.uk/pay-2/council-tax-expenditure diff --git a/pyproject.toml b/pyproject.toml index eadfb41a..8622b327 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ dependencies = [ "numpy>=1.24", "policyengine==6.1.2", "policyengine-us==2.38.6", - "policyengine-uk==2.125.1", + "policyengine-uk==2.126.7", "anthropic>=0.115.1", "google-genai>=2.10.0", ] @@ -42,7 +42,7 @@ policybench = "policybench.cli:main" include = ["policybench*"] [tool.setuptools.package-data] -policybench = ["benchmark_specs.json", "population_weights.json"] +policybench = ["benchmark_specs.json", "population_weights.json", "uk_audit_councils.csv"] [tool.ruff] line-length = 88 diff --git a/tests/test_supervisor.py b/tests/test_supervisor.py index afb1a95f..1b4240b9 100644 --- a/tests/test_supervisor.py +++ b/tests/test_supervisor.py @@ -1592,6 +1592,41 @@ def __init__(self, cmd, **kwargs): ) +def test_workers_request_a_named_output_set(uk_manifest, tmp_path, monkeypatch): + supervisor = make_supervisor(uk_manifest, tmp_path, program_set="uk_audit") + launched = [] + + class FakePopen: + def __init__(self, cmd, **kwargs): + launched.append(cmd) + + monkeypatch.setattr(subprocess, "Popen", FakePopen) + supervisor._spawn(0) + + (cmd,) = launched + assert cmd[cmd.index("--program-set") + 1] == "uk_audit" + audit = get_programs("uk", "uk_audit") + assert supervisor._expected_outputs_for_scenario(0) == audit + assert supervisor.initial_request_variables == audit + assert ( + supervisor.workload["output_set_sha256"] + == hashlib.sha256("\n".join(sorted(audit)).encode("utf-8")).hexdigest() + ) + + +def test_default_output_set_adds_no_worker_flag(uk_manifest, tmp_path, monkeypatch): + supervisor = make_supervisor(uk_manifest, tmp_path) + launched = [] + + class FakePopen: + def __init__(self, cmd, **kwargs): + launched.append(cmd) + + monkeypatch.setattr(subprocess, "Popen", FakePopen) + supervisor._spawn(0) + assert "--program-set" not in launched[0] + + def test_us_workload_fingerprint_is_unchanged_by_country_awareness(manifest, tmp_path): supervisor = make_supervisor(manifest, tmp_path) assert ( diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py new file mode 100644 index 00000000..27687ef8 --- /dev/null +++ b/tests/test_uk_audit.py @@ -0,0 +1,373 @@ +"""Tests for the stratified UK audit sample (policybench.uk_audit).""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policybench import uk_audit +from policybench.config import get_programs +from policybench.prompts import describe_household +from policybench.scenarios import Person, Scenario, scenario_from_dict, scenario_to_dict + +REGIONS = ( + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", + "WALES", + "SCOTLAND", + "NORTHERN_IRELAND", +) + + +def _frames(n_households: int = 3000, seed: int = 0): + """Synthetic transfer frames with every structure the strata select.""" + rng = np.random.default_rng(seed) + people, households = [], [] + person_id = 0 + for household_id in range(n_households): + kind = household_id % 6 + region = REGIONS[int(rng.integers(len(REGIONS)))] + if kind == 0: # pensioner couple + ages = [int(rng.integers(67, 90)), int(rng.integers(67, 90))] + elif kind == 1: # family + ages = [int(rng.integers(25, 50)), int(rng.integers(25, 50)), 3, 9] + elif kind == 2: # single parent + ages = [int(rng.integers(20, 45)), int(rng.integers(0, 15))] + elif kind == 3: # working-age couple + ages = [int(rng.integers(25, 64)), int(rng.integers(25, 64))] + elif kind == 4: # single adult + ages = [int(rng.integers(20, 64))] + else: # single pensioner + ages = [int(rng.integers(67, 95))] + low = rng.random() < 0.4 + for position, age in enumerate(ages): + adult = age >= 18 + earnings = 0.0 + if adult and age < 67: + earnings = float( + rng.uniform(0, 12_000) if low else rng.uniform(20_000, 130_000) + ) + self_employment = 0.0 + if adult and age < 67 and rng.random() < 0.08: + self_employment = float(rng.uniform(-8_000, -100)) + people.append( + { + "person_id": person_id, + "person_household_id": household_id, + "person_benunit_id": household_id, + "age": age, + "employment_income_before_lsr": earnings, + "self_employment_income": self_employment, + "private_pension_income": float(rng.uniform(0, 4_000)) + if age >= 67 + else 0.0, + "property_income": 0.0, + "savings_interest_income": 0.0, + "dividend_income": 0.0, + "miscellaneous_income": 0.0, + "gift_aid": 100.0 if position == 0 else 0.0, + "date_of_birth": (2026 - age) * 10000 + 701, + "pip_dl_category": "NONE", + } + ) + person_id += 1 + households.append( + { + "household_id": household_id, + "household_weight": float(rng.uniform(1, 100)), + "region": region, + "tenure_type": "RENT_PRIVATELY" + if rng.random() < 0.4 + else "OWNED_OUTRIGHT", + "rent": float(rng.uniform(5_000, 15_000)), + "savings": float( + rng.uniform(0, 10_000) if low else rng.uniform(20_000, 200_000) + ), + "council_tax_band": str(rng.choice(list("ABCDEFGHI"))), + } + ) + return pd.DataFrame(people), pd.DataFrame(households) + + +@pytest.fixture(scope="module") +def sample(): + person_df, household_df = _frames() + return uk_audit.stratified_uk_audit_scenarios( + person_df, household_df, dataset_year=2025 + ) + + +def _stratum(scenarios, name): + return [s for s in scenarios if s.metadata["stratum"] == name] + + +def test_strata_counts_and_distinct_households(sample): + expected = {s.name: s.count for s in uk_audit.UK_AUDIT_STRATA} + assert dict(uk_audit.stratum_counts(sample)) == expected + assert len(sample) == sum(expected.values()) == 200 + ids = [s.metadata["household_id"] for s in sample] + assert len(ids) == len(set(ids)) + assert [s.id for s in sample] == [f"scenario_{i:03d}" for i in range(len(sample))] + + +def test_sample_is_deterministic(): + person_df, household_df = _frames() + first = uk_audit.stratified_uk_audit_scenarios(person_df, household_df) + second = uk_audit.stratified_uk_audit_scenarios(person_df, household_df) + assert [scenario_to_dict(s) for s in first] == [scenario_to_dict(s) for s in second] + + +def test_a_later_stratum_does_not_change_earlier_draws(): + person_df, household_df = _frames() + strata = uk_audit.UK_AUDIT_STRATA + full = uk_audit.stratified_uk_audit_scenarios( + person_df, household_df, strata=strata + ) + shorter = uk_audit.stratified_uk_audit_scenarios( + person_df, household_df, strata=strata[:-1] + ) + assert [scenario_to_dict(s) for s in shorter] == [ + scenario_to_dict(s) for s in full[: len(shorter)] + ] + + +def test_every_household_is_in_great_britain_with_a_council(sample): + for scenario in sample: + assert scenario.state != "NORTHERN_IRELAND" + council = scenario.household_inputs["local_authority"] + regional = {c.local_authority: c for c in uk_audit.COUNCILS[scenario.state]} + assert council in regional + band = scenario.household_inputs["council_tax_band"] + charge = regional[council].band_d * uk_audit.band_ratio(scenario.state, band) + if len(scenario.adults) == 1: + charge *= 1 - uk_audit.SINGLE_PERSON_DISCOUNT + assert scenario.household_inputs["council_tax"] == round(charge) + if band == "I": + assert scenario.state == "WALES" + + +def test_private_renters_are_placed_in_their_councils_rental_area(sample): + for scenario in sample: + inputs = scenario.household_inputs + if inputs.get("tenure_type") == "RENT_PRIVATELY": + regional = {c.local_authority: c for c in uk_audit.COUNCILS[scenario.state]} + assert inputs["brma"] == regional[inputs["local_authority"]].brma + + +def test_everyone_67_or_over_has_a_state_pension_outside_control(sample): + for scenario in sample: + for adult in scenario.adults: + stated = adult.inputs.get("state_pension_reported") + if scenario.metadata["stratum"] == "control": + assert stated is None + elif adult.age >= uk_audit.PENSIONER_MIN_AGE: + assert stated and stated > 0 + else: + assert stated is None + + +def test_pension_strata(sample): + for name in ("pension_age", "pension_age_low_income"): + for scenario in _stratum(sample, name): + assert all(a.age >= uk_audit.PENSIONER_MIN_AGE for a in scenario.adults) + assert not scenario.children + tenures = { + s.household_inputs.get("tenure_type") + for s in _stratum(sample, "pension_age") + + _stratum(sample, "pension_age_low_income") + } + assert tenures & set(uk_audit.SOCIAL_TENURES) + + +def test_carers_stratum(sample): + carers = _stratum(sample, "carers") + assert carers + for scenario in carers: + assert len(scenario.adults) == 2 + assert scenario.state not in ("SCOTLAND", "NORTHERN_IRELAND") + hours = [a.inputs.get("care_hours") for a in scenario.adults] + assert sum(h is not None for h in hours) == 1 + assert {h for h in hours if h is not None} <= set(uk_audit.CARE_HOURS) + + +def test_childcare_costs_only_for_children_eleven_or_under(sample): + for scenario in sample: + for child in scenario.children: + cost = child.inputs.get("childcare_expenses") + if scenario.metadata["stratum"] == "childcare" and child.age <= 11: + assert cost and cost > 0 + assert ( + child.inputs["tax_free_childcare_uses_qualifying_provider"] is True + ) + else: + assert cost is None + + +def test_gift_aid_only_in_control(sample): + for scenario in sample: + has_gift_aid = any(p.inputs.get("gift_aid") for p in scenario.all_people) + if has_gift_aid: + assert scenario.metadata["stratum"] == "control" + + +def test_low_income_strata(sample): + for name in ( + "scotland_low_income_families", + "pension_age_low_income", + "private_renters_low_income", + ): + for scenario in _stratum(sample, name): + assert scenario.household_inputs.get("savings", 0) < uk_audit.LOW_SAVINGS + + +def test_scenarios_round_trip_through_the_manifest(sample): + for scenario in sample: + assert scenario_from_dict(scenario_to_dict(scenario)) == scenario + + +@settings(max_examples=300, deadline=None) +@given( + dob=st.integers(min_value=19200101, max_value=19600405), + seed=st.integers(min_value=0, max_value=2**32 - 1), +) +def test_state_pension_is_within_the_scheme_bounds(dob, seed): + """Every drawn State Pension lies between 60% of its scheme's flat rate + and the flat rate plus the largest top-up, on the scheme the date of + birth puts the person on.""" + person = Person( + name="adult1", age=70, employment_income=0, inputs={"date_of_birth": dob} + ) + annual = uk_audit.draw_state_pension(person, np.random.default_rng(seed)) + weekly = annual / uk_audit.WEEKS_PER_YEAR + if dob >= uk_audit.NEW_STATE_PENSION_FIRST_BIRTH_DATE: + low, high = ( + 0.60 * uk_audit.NEW_STATE_PENSION_WEEKLY, + uk_audit.NEW_STATE_PENSION_WEEKLY + 60, + ) + else: + low, high = ( + 0.60 * uk_audit.BASIC_STATE_PENSION_WEEKLY, + uk_audit.BASIC_STATE_PENSION_WEEKLY + 120, + ) + assert low - 0.01 <= weekly <= high + 0.01 + + +@given(region=st.sampled_from(["SCOTLAND", "WALES", "LONDON"])) +def test_band_ratios_increase_with_band(region): + table = ( + uk_audit.SCOTLAND_BAND_RATIOS + if region == "SCOTLAND" + else uk_audit.WALES_BAND_RATIOS + if region == "WALES" + else uk_audit.ENGLAND_BAND_RATIOS + ) + ratios = [uk_audit.band_ratio(region, band) for band in sorted(table)] + assert ratios == sorted(ratios) + assert uk_audit.band_ratio(region, "D") == 1 + + +def test_valid_band_maps_band_i_outside_wales(): + assert uk_audit.valid_band("WALES", "I") == "I" + assert uk_audit.valid_band("LONDON", "I") == "H" + assert uk_audit.valid_band("SCOTLAND", "I") == "H" + + +def test_council_table_is_sourced_and_covers_every_gb_region(): + for region in REGIONS: + if region == "NORTHERN_IRELAND": + continue + assert uk_audit.COUNCILS[region] + for councils in uk_audit.COUNCILS.values(): + for council in councils: + assert council.band_d > 0 + assert council.band_d_source + simulated = { + c.local_authority for cs in uk_audit.COUNCILS.values() for c in cs + } & set(uk_audit.SIMULATED_WORKING_AGE_CTR_COUNCILS) + assert simulated == set(uk_audit.SIMULATED_WORKING_AGE_CTR_COUNCILS) + + +def test_council_table_names_exist_in_policyengine_uk(): + locations = pytest.importorskip( + "policyengine_uk.variables.household.demographic.locations" + ) + for region, councils in uk_audit.COUNCILS.items(): + for council in councils: + assert council.local_authority in locations.LocalAuthority.__members__ + assert council.brma in locations.BRMAName.__members__ + + +def test_uk_audit_outputs(): + programs = get_programs("uk", "uk_audit") + assert programs[:7] == get_programs("uk") + assert programs[7:] == [ + "high_income_child_benefit_charge", + "carers_allowance", + "attendance_allowance", + "housing_benefit", + "council_tax_reduction", + "tax_free_childcare", + "scottish_child_payment", + ] + + +def test_uk_audit_outputs_map_to_policyengine_uk_variables(): + pytest.importorskip("policyengine_uk") + from policyengine_uk import CountryTaxBenefitSystem + + from policybench.ground_truth import _pe_variable_for_output + + variables = CountryTaxBenefitSystem().variables + for program in get_programs("uk", "uk_audit"): + assert _pe_variable_for_output(program, "uk") in variables + + +def test_prompt_states_the_new_facts(): + scenario = Scenario( + id="scenario_000", + state="WALES", + filing_status=None, + adults=[ + Person( + name="adult1", + age=78, + employment_income=0, + inputs={ + "state_pension_reported": 12_548.0, + "aa_category": "LOWER", + "date_of_birth": 19480601, + }, + ), + Person( + name="adult2", age=60, employment_income=0, inputs={"care_hours": 40.0} + ), + ], + household_inputs={ + "tenure_type": "RENT_FROM_HA", + "rent": 6_000.0, + "local_authority": "CARDIFF", + "council_tax_band": "C", + "council_tax": 1_786.0, + }, + country="uk", + ) + text = describe_household(scenario) + assert "State Pension (amount for 2026-27): £12,548" in text + assert "Attendance Allowance award: lower rate" in text + assert "hours a week spent caring for the other adult in the household: 40" in text + assert "£40" not in text + assert "tenure: renting from a housing association" in text + assert "council (local authority): Cardiff" in text + assert "after the single person discount" in text + situation = scenario.to_pe_uk_situation() + assert situation["people"]["adult1"]["state_pension_reported"] == {"2026": 12_548.0} diff --git a/uv.lock b/uv.lock index c294da04..5baed6ed 100644 --- a/uv.lock +++ b/uv.lock @@ -2510,7 +2510,7 @@ requires-dist = [ { name = "numpy", specifier = ">=1.24" }, { name = "pandas", specifier = ">=2.0" }, { name = "policyengine", specifier = "==6.1.2" }, - { name = "policyengine-uk", specifier = "==2.125.1" }, + { name = "policyengine-uk", specifier = "==2.126.7" }, { name = "policyengine-us", specifier = "==2.38.6" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" }, @@ -2566,7 +2566,7 @@ wheels = [ [[package]] name = "policyengine-uk" -version = "2.125.1" +version = "2.126.7" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -2574,9 +2574,9 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/3b/c7b4eaa2b80df7e520d0576399cf1c2cec6e29bdf4ecb0104496cfa0b946/policyengine_uk-2.125.1.tar.gz", hash = "sha256:96e0354c3e8d19cd9d4e7dc5ba6c3611f1305a589b63c5f065441df346c33ed5", size = 2112669, upload-time = "2026-10-10T00:21:19.345Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/cb/097fde0e766dfd87e341d542579eeea372b08de63c3604d2e66280b09e47/policyengine_uk-2.126.7.tar.gz", hash = "sha256:d07ce8e2f6395e134be93ce992c68abce39e7e4ec4be977f821efce173215d1b", size = 2204097, upload-time = "2026-10-11T13:00:14.912Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/3a/1d90bad4346d1a8c2715be211ed3d3d7f48917da0d90405b8257b7304091/policyengine_uk-2.125.1-py3-none-any.whl", hash = "sha256:a0983b4cef868467578a4fd0d163a977a0da8f710dd5194ec01c75f66d512c5a", size = 3305979, upload-time = "2026-10-10T00:21:17.194Z" }, + { url = "https://files.pythonhosted.org/packages/56/95/28f746372c6d883c7bf1b9e860946869b87b4865d26e2ff3e338d6868b1d/policyengine_uk-2.126.7-py3-none-any.whl", hash = "sha256:a58857e082cf1b30ae16d1032f361f5a65d9ca1b45601c584da0c95a95aeed11", size = 3428729, upload-time = "2026-10-11T13:00:11.773Z" }, ] [[package]] From 6b4ff53b5f8c3d67eb19acdbb761d426e502ad89 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:25:29 -0400 Subject: [PATCH 2/8] Select the low-income strata on capital, not only savings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pension Credit and Housing Benefit count corporate financial wealth and property other than the home as capital, so a household with little savings but large holdings was not low-income for the means tests the strata exist to exercise. The strata now require savings, corporate wealth and other property together under £16,000. Co-Authored-By: Claude Opus 5.5 --- policybench/uk_audit.py | 27 +++++++++++++++++++-------- tests/test_uk_audit.py | 6 +++++- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/policybench/uk_audit.py b/policybench/uk_audit.py index cc29c702..74a18fa6 100644 --- a/policybench/uk_audit.py +++ b/policybench/uk_audit.py @@ -185,7 +185,7 @@ class Household: adult_earnings: tuple[float, ...] adult_self_employment: tuple[float, ...] adult_income: tuple[float, ...] - savings: float + capital: float def household_features( @@ -231,7 +231,10 @@ def household_features( adult_income=tuple( float(v) for v in adults[income_columns].sum(axis=1) ), - savings=float(getattr(row, "savings", 0.0) or 0.0), + capital=sum( + float(getattr(row, column, 0.0) or 0.0) + for column in CAPITAL_COLUMNS + ), ) ) return features @@ -266,15 +269,23 @@ def select_scotland(h: Household) -> bool: # Low income: every adult's transfer-data income (before any State Pension) -# below £15,000 and household savings below £16,000 (the Universal Credit and +# below £15,000 and household capital below £16,000 (the Universal Credit and # Housing Benefit capital limit), so the means-tested benefits have cells to # test. LOW_INCOME = 15_000 -LOW_SAVINGS = 16_000 +LOW_CAPITAL = 16_000 +# Household holdings the prompt states that means tests can count as capital: +# savings, corporate financial wealth, and property other than the home. +CAPITAL_COLUMNS = ( + "savings", + "corporate_wealth", + "other_residential_property_value", + "non_residential_property_value", +) def _low_income(h: Household) -> bool: - return max(h.adult_income) < LOW_INCOME and h.savings < LOW_SAVINGS + return max(h.adult_income) < LOW_INCOME and h.capital < LOW_CAPITAL def select_scotland_low_income_families(h: Household) -> bool: @@ -639,7 +650,7 @@ def _band(scenario: Scenario, rng: np.random.Generator) -> str: "scotland_low_income_families", 10, "Scotland: a child under 16, every adult's income under £15,000 and " - "savings under £16,000", + "capital under £16,000", select_scotland_low_income_families, overlay_scotland, ), @@ -654,7 +665,7 @@ def _band(scenario: Scenario, rng: np.random.Generator) -> str: "pension_age_low_income", 20, "Every adult 67 or over, no children, every adult's income other than " - "the State Pension under £15,000, savings under £16,000", + "the State Pension under £15,000, capital under £16,000", select_pension_age_low_income, overlay_pension_age, ), @@ -705,7 +716,7 @@ def _band(scenario: Scenario, rng: np.random.Generator) -> str: "private_renters_low_income", 10, "Working-age private renters, every adult's income under £15,000 and " - "savings under £16,000", + "capital under £16,000", select_private_renters_low_income, overlay_private_renters, ), diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py index 27687ef8..17a6df8f 100644 --- a/tests/test_uk_audit.py +++ b/tests/test_uk_audit.py @@ -227,7 +227,11 @@ def test_low_income_strata(sample): "private_renters_low_income", ): for scenario in _stratum(sample, name): - assert scenario.household_inputs.get("savings", 0) < uk_audit.LOW_SAVINGS + capital = sum( + scenario.household_inputs.get(column, 0) + for column in uk_audit.CAPITAL_COLUMNS + ) + assert capital < uk_audit.LOW_CAPITAL def test_scenarios_round_trip_through_the_manifest(sample): From 24f84f506c9c81876e43dcc085e191c9dd2fe5a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:28:15 -0400 Subject: [PATCH 3/8] Let the full-run export keep a named output set; fix a json shadowing in the CLI export-full-run kept only the headline outputs, so a uk_audit run lost its seven new outputs before flagging. It now takes --program-set (default headline, unchanged). The uk-audit-sample branch imported json inside main(), which made json local to the whole function and broke other commands. Co-Authored-By: Claude Opus 5.5 --- policybench/cli.py | 8 ++++++-- policybench/full_run_export.py | 26 ++++++++++++++++++-------- tests/test_export_full_run.py | 18 +++++++++++++++++- tests/test_uk_audit.py | 17 +++++++++++++++++ 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/policybench/cli.py b/policybench/cli.py index efef6a43..b3850c56 100644 --- a/policybench/cli.py +++ b/policybench/cli.py @@ -542,6 +542,11 @@ def main(): action="store_true", help="Only write the combined payload under the run directory", ) + export_parser.add_argument( + "--program-set", + default=None, + help="Output set the run asked for (default: headline)", + ) # Failure audit (Codex-backed classifier) audit_prepare_parser = subparsers.add_parser( @@ -1106,8 +1111,6 @@ def main(): enable_cache() if args.command == "uk-audit-sample": - import json - from policybench.uk_audit import UK_AUDIT_SEED, write_uk_audit_sample meta = write_uk_audit_sample( @@ -1403,6 +1406,7 @@ def main(): reference_digest=parse_reference_digest_args(args.reference_digest), app_data_output=args.app_data_output, skip_app_data=args.skip_app_data, + program_set=args.program_set, ) except (FileNotFoundError, ReferenceProvenanceError) as exc: raise SystemExit(str(exc)) from exc diff --git a/policybench/full_run_export.py b/policybench/full_run_export.py index ea1006f3..d993af7b 100644 --- a/policybench/full_run_export.py +++ b/policybench/full_run_export.py @@ -46,15 +46,18 @@ class ReferenceProvenanceError(ValueError): ) -def _canonical_output_ids(country: str) -> set[str]: - return set(get_output_ids(country, "headline")) +def _canonical_output_ids(country: str, program_set: str | None = None) -> set[str]: + return set(get_output_ids(country, program_set or "headline")) -def _filter_to_canonical_outputs(frame: pd.DataFrame, country: str) -> pd.DataFrame: - """Drop source-run rows outside the canonical headline output scope.""" +def _filter_to_canonical_outputs( + frame: pd.DataFrame, country: str, program_set: str | None = None +) -> pd.DataFrame: + """Drop source-run rows outside the run's output set (the headline set + unless the run names another, such as the internal ``uk_audit`` set).""" if frame.empty or "variable" not in frame.columns: return frame - canonical = _canonical_output_ids(country) + canonical = _canonical_output_ids(country, program_set) return frame[frame["variable"].map(output_group_id).isin(canonical)].reset_index( drop=True ) @@ -405,7 +408,12 @@ def reference_digest_for_country( return reference_digest -def export_country(country_dir: Path, *, reference_digest: str | None = None) -> dict: +def export_country( + country_dir: Path, + *, + reference_digest: str | None = None, + program_set: str | None = None, +) -> dict: """Write analysis artifacts and dashboard payload for one country run. Reference provenance is verified strictly: the sidecar must carry the @@ -446,8 +454,8 @@ def export_country(country_dir: Path, *, reference_digest: str | None = None) -> predictions, load_case_reference_explanations(country_dir), ) - ground_truth = _filter_to_canonical_outputs(ground_truth, country) - predictions = _filter_to_canonical_outputs(predictions, country) + ground_truth = _filter_to_canonical_outputs(ground_truth, country, program_set) + predictions = _filter_to_canonical_outputs(predictions, country, program_set) # Outputs whose reference depends on an input the data never carried are # scored for no model; they stay in the payload marked scored=false. @@ -588,6 +596,7 @@ def export_full_run( app_data_output: str | Path = "app/src/data.json", skip_app_data: bool = False, reference_digest: ReferenceDigests = None, + program_set: str | None = None, ) -> dict: """Export per-country and combined frontend artifacts from a full run. @@ -614,6 +623,7 @@ def export_full_run( reference_digest=reference_digest_for_country( reference_digest, country, len(selected_countries) ), + program_set=program_set, ) for country in selected_countries } diff --git a/tests/test_export_full_run.py b/tests/test_export_full_run.py index c24d306a..fe1dee67 100644 --- a/tests/test_export_full_run.py +++ b/tests/test_export_full_run.py @@ -762,7 +762,7 @@ def test_export_full_run_pins_each_legacy_country_separately(tmp_path, monkeypat } seen: dict[str, str | None] = {} - def fake_export_country(country_dir, *, reference_digest=None): + def fake_export_country(country_dir, *, reference_digest=None, program_set=None): seen[country_dir.name] = reference_digest return {"country": country_dir.name, "modelStats": []} @@ -815,3 +815,19 @@ def test_export_full_run_mixes_legacy_and_digest_sidecars(tmp_path): manifest_reference_sha256=reference_digest_for_country(pins, country, 2), ) assert country in bundles + + +def test_the_uk_audit_output_set_keeps_its_outputs(): + from policybench.full_run_export import _filter_to_canonical_outputs + + frame = pd.DataFrame( + { + "scenario_id": ["scenario_000"] * 3, + "variable": ["income_tax", "carers_allowance", "not_an_output"], + "value": [1.0, 2.0, 3.0], + } + ) + headline = _filter_to_canonical_outputs(frame, "uk") + assert headline["variable"].tolist() == ["income_tax"] + audit = _filter_to_canonical_outputs(frame, "uk", "uk_audit") + assert audit["variable"].tolist() == ["income_tax", "carers_allowance"] diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py index 17a6df8f..7551dedc 100644 --- a/tests/test_uk_audit.py +++ b/tests/test_uk_audit.py @@ -375,3 +375,20 @@ def test_prompt_states_the_new_facts(): assert "after the single person discount" in text situation = scenario.to_pe_uk_situation() assert situation["people"]["adult1"]["state_pension_reported"] == {"2026": 12_548.0} + + +def test_uk_audit_sample_cli(tmp_path, monkeypatch, capsys): + from policybench import cli + + calls = [] + + def fake_write(output, seed): + calls.append((str(output), seed)) + return {"counts": {"control": 15}} + + monkeypatch.setattr(uk_audit, "write_uk_audit_sample", fake_write) + out = tmp_path / "scenarios.csv" + monkeypatch.setattr("sys.argv", ["policybench", "uk-audit-sample", "-o", str(out)]) + cli.main() + assert calls == [(str(out), uk_audit.UK_AUDIT_SEED)] + assert "UK audit sample saved" in capsys.readouterr().out From 21a0ae622c97958f381e27aabeabc97ceaf04911 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:30:09 -0400 Subject: [PATCH 4/8] =?UTF-8?q?Keep=20UK=20audit=20birth=20dates=20before?= =?UTF-8?q?=20the=20fiscal=20year;=20print=20UK=20losses=20as=20-=C2=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A baby aged 0 could be given a date of birth after 6 April of the fiscal year, contradicting the prompt's whole-year facts (Child Benefit would start at the birth while the reference pays the year). Such dates move back a year. UK negative amounts print as -£6,756 rather than £-6,756. Co-Authored-By: Claude Opus 5.5 --- policybench/prompts.py | 6 +++++- policybench/uk_audit.py | 30 ++++++++++++++++++++++++++++-- tests/test_uk_audit.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/policybench/prompts.py b/policybench/prompts.py index 5b183730..447d84c1 100644 --- a/policybench/prompts.py +++ b/policybench/prompts.py @@ -431,7 +431,11 @@ def _format_input_line(field: str, value, country: str = "us") -> str: return f"- {label}: {float(value):,.0f}" if field.endswith(RATE_OR_RATIO_FIELD_SUFFIXES): return f"- {label}: {float(value):,.4g}" - return f"- {label}: {_currency_symbol(country)}{float(value):,.0f}" + amount = float(value) + if country == "uk" and amount < 0: + # "-£6,756", not "£-6,756". + return f"- {label}: -{_currency_symbol(country)}{abs(amount):,.0f}" + return f"- {label}: {_currency_symbol(country)}{amount:,.0f}" def _person_heading(person: Person, country: str = "us") -> str: diff --git a/policybench/uk_audit.py b/policybench/uk_audit.py index 74a18fa6..be8874bf 100644 --- a/policybench/uk_audit.py +++ b/policybench/uk_audit.py @@ -737,6 +737,24 @@ def _band(scenario: Scenario, rng: np.random.Generator) -> str: ) +def _born_before_the_year(person: Person, year: int) -> Person: + """Keep every date of birth before the fiscal year starts. + + The transfer path places a person within their year of age with a seeded + date of birth, so a baby aged 0 can be born after 6 April of the fiscal + year. The prompt says its facts hold for the whole year, which a birth + during the year contradicts: Child Benefit and the Universal Credit child + element would start at the birth, while the reference pays the whole + year. A later date moves back one year, so the child is under one at the + start of the year. + """ + dob = _date_of_birth(person) + year_start = year * 10000 + 406 + if dob is None or dob < year_start: + return person + return _with_person_inputs(person, date_of_birth=float(dob - 10000)) + + def _scenario_for( household_id: int, person_df: pd.DataFrame, @@ -753,9 +771,17 @@ def _scenario_for( adults, children = [], [] for _, row in members.iterrows(): if int(row["age"]) >= 18: - adults.append(_build_uk_person(row, f"adult{len(adults) + 1}")) + adults.append( + _born_before_the_year( + _build_uk_person(row, f"adult{len(adults) + 1}"), year + ) + ) else: - children.append(_build_uk_person(row, f"child{len(children) + 1}")) + children.append( + _born_before_the_year( + _build_uk_person(row, f"child{len(children) + 1}"), year + ) + ) metadata: dict[str, Any] = { "household_id": int(household_id), "stratum": stratum, diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py index 7551dedc..d9f2532c 100644 --- a/tests/test_uk_audit.py +++ b/tests/test_uk_audit.py @@ -392,3 +392,34 @@ def fake_write(output, seed): cli.main() assert calls == [(str(out), uk_audit.UK_AUDIT_SEED)] assert "UK audit sample saved" in capsys.readouterr().out + + +def test_no_one_is_born_during_the_fiscal_year(sample): + for scenario in sample: + for person in scenario.all_people: + dob = person.inputs.get("date_of_birth") + if dob is not None: + assert int(dob) < 20260406 + + +def test_born_during_the_year_moves_back_a_year(): + baby = Person( + name="child1", age=0, employment_income=0, inputs={"date_of_birth": 20260710} + ) + assert ( + uk_audit._born_before_the_year(baby, 2026).inputs["date_of_birth"] == 20250710 + ) + older = Person( + name="child1", age=0, employment_income=0, inputs={"date_of_birth": 20260101} + ) + assert uk_audit._born_before_the_year(older, 2026) == older + + +def test_uk_losses_render_with_the_sign_first(): + from policybench.prompts import _format_input_line + + line = _format_input_line("self_employment_income", -6756.0, country="uk") + assert line == "- self-employment income: -£6,756" + assert _format_input_line("self_employment_income", -50.0, country="us").endswith( + "$-50" + ) From 85a34ede61f7cd86c24a8561ef4e55b4af70437d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:36:41 -0400 Subject: [PATCH 5/8] Keep the transfer path stating one State Pension; update UK test expectations The transfer path states the State Pension PE-UK computes (state_pension), so it now skips state_pension_reported, which the audit sample states instead. Tests follow the new tenure wording and use a still-hidden input for the rejection case. Co-Authored-By: Claude Opus 5.5 --- policybench/scenarios.py | 5 +++++ tests/test_eval_no_tools.py | 2 +- tests/test_scenarios.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 31dbf412..62a2c65c 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -1647,6 +1647,11 @@ def _extract_uk_person_inputs(row: pd.Series) -> dict[str, Any]: UK_EDUCATION_PROMPT_AGES ): continue + # The transfer path states the State Pension PE-UK computes + # (state_pension, in UK_COMPUTED_PERSON_INPUTS); stating the reported + # amount as well would show two State Pension lines. + if col == "state_pension_reported": + continue promptable = _uk_promptable_value(value) if promptable is not None: inputs[col] = promptable diff --git a/tests/test_eval_no_tools.py b/tests/test_eval_no_tools.py index d9d5a112..e30980fb 100644 --- a/tests/test_eval_no_tools.py +++ b/tests/test_eval_no_tools.py @@ -436,7 +436,7 @@ def test_no_tools_prompt_supports_uk_households(uk_scenario): assert "head:" not in prompt_lower assert "household total annual uk income tax liability" in prompt_lower assert "excluding capital gains tax" in prompt_lower - assert "tenure: rent privately" in prompt_lower + assert "tenure: renting privately" in prompt_lower assert "£42,000" in prompt assert "£14,400" in prompt assert "currency amount" in prompt_lower diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 70826026..d0310257 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1511,7 +1511,7 @@ def test_uk_situation_rejects_inputs_the_prompt_cannot_show(): name="adult1", age=70, employment_income=0.0, - inputs={"state_pension_reported": 9_000.0}, + inputs={"universal_credit_reported": 9_000.0}, ) ], ) From 95caec817b3742648c5cf015b291ed0b4f490c07 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 11:45:53 -0400 Subject: [PATCH 6/8] Move a baby born during the fiscal year to a date that keeps it aged 0 policyengine-uk takes age as the age on 6 October and rejects a date of birth that disagrees. The previous move (back a full year) made a baby aged 1. The date now moves back 182 days within 7 October to 5 April, and a test runs the moved baby through the engine. Co-Authored-By: Claude Opus 5.5 --- policybench/uk_audit.py | 23 +++++++++------- tests/test_uk_audit.py | 58 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/policybench/uk_audit.py b/policybench/uk_audit.py index be8874bf..6b57ff1a 100644 --- a/policybench/uk_audit.py +++ b/policybench/uk_audit.py @@ -26,6 +26,7 @@ import csv from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace +from datetime import date, timedelta from pathlib import Path from typing import Any @@ -741,18 +742,22 @@ def _born_before_the_year(person: Person, year: int) -> Person: """Keep every date of birth before the fiscal year starts. The transfer path places a person within their year of age with a seeded - date of birth, so a baby aged 0 can be born after 6 April of the fiscal - year. The prompt says its facts hold for the whole year, which a birth - during the year contradicts: Child Benefit and the Universal Credit child - element would start at the birth, while the reference pays the whole - year. A later date moves back one year, so the child is under one at the - start of the year. + date of birth, and policyengine-uk takes age as the age on 6 October. So a + baby aged 0 can be born after 6 April of the fiscal year. The prompt says + its facts hold for the whole year, which a birth during the year + contradicts: Child Benefit and the Universal Credit child element would + start at the birth, while the reference pays the whole year. Such a date + moves back 182 days, kept within 7 October of the previous year to 5 April + of this one, so the baby is still aged 0 on 6 October and is born before + the year starts. """ dob = _date_of_birth(person) - year_start = year * 10000 + 406 - if dob is None or dob < year_start: + if dob is None or dob < year * 10000 + 406: return person - return _with_person_inputs(person, date_of_birth=float(dob - 10000)) + born = date(dob // 10000, dob // 100 % 100, dob % 100) - timedelta(days=182) + born = min(max(born, date(year - 1, 10, 7)), date(year, 4, 5)) + moved = born.year * 10000 + born.month * 100 + born.day + return _with_person_inputs(person, date_of_birth=float(moved)) def _scenario_for( diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py index d9f2532c..22ca51ba 100644 --- a/tests/test_uk_audit.py +++ b/tests/test_uk_audit.py @@ -402,12 +402,25 @@ def test_no_one_is_born_during_the_fiscal_year(sample): assert int(dob) < 20260406 -def test_born_during_the_year_moves_back_a_year(): +def test_born_during_the_year_moves_back_within_age_zero(): baby = Person( name="child1", age=0, employment_income=0, inputs={"date_of_birth": 20260710} ) + moved = int(uk_audit._born_before_the_year(baby, 2026).inputs["date_of_birth"]) + assert moved == 20260109 + late = Person( + name="child1", age=0, employment_income=0, inputs={"date_of_birth": 20261006} + ) assert ( - uk_audit._born_before_the_year(baby, 2026).inputs["date_of_birth"] == 20250710 + int(uk_audit._born_before_the_year(late, 2026).inputs["date_of_birth"]) + == 20260405 + ) + first = Person( + name="child1", age=0, employment_income=0, inputs={"date_of_birth": 20260406} + ) + assert ( + int(uk_audit._born_before_the_year(first, 2026).inputs["date_of_birth"]) + == 20251007 ) older = Person( name="child1", age=0, employment_income=0, inputs={"date_of_birth": 20260101} @@ -415,6 +428,22 @@ def test_born_during_the_year_moves_back_a_year(): assert uk_audit._born_before_the_year(older, 2026) == older +@settings(max_examples=200, deadline=None) +@given(day=st.integers(min_value=0, max_value=183)) +def test_moved_birth_dates_keep_age_zero_on_6_october(day): + """policyengine-uk takes age as the age on 6 October: a moved date must + be after 6 October 2025 and before 6 April 2026.""" + from datetime import date, timedelta + + born = date(2026, 4, 6) + timedelta(days=day) + dob = born.year * 10000 + born.month * 100 + born.day + baby = Person( + name="child1", age=0, employment_income=0, inputs={"date_of_birth": dob} + ) + moved = int(uk_audit._born_before_the_year(baby, 2026).inputs["date_of_birth"]) + assert 20251006 < moved < 20260406 + + def test_uk_losses_render_with_the_sign_first(): from policybench.prompts import _format_input_line @@ -423,3 +452,28 @@ def test_uk_losses_render_with_the_sign_first(): assert _format_input_line("self_employment_income", -50.0, country="us").endswith( "$-50" ) + + +def test_a_moved_baby_passes_policyengine_uks_age_check(): + pytest.importorskip("policyengine_uk") + from policyengine_uk import Simulation + + baby = uk_audit._born_before_the_year( + Person( + name="child1", + age=0, + employment_income=0, + inputs={"date_of_birth": 20260710}, + ), + 2026, + ) + scenario = Scenario( + id="scenario_000", + state="LONDON", + filing_status=None, + adults=[Person(name="adult1", age=30, employment_income=20_000)], + children=[baby], + country="uk", + ) + sim = Simulation(situation=scenario.to_pe_uk_situation()) + assert float(sim.calculate("child_benefit", 2026).sum()) > 0 From c2163847e2692cf3aecbd179c1e2335006c6380f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 14:24:20 -0400 Subject: [PATCH 7/8] State the adults of a UK household with children as parents The prompt says the children are the adults' dependents, but PE-UK reads is_parent as an input that defaults to false, and Tax-Free Childcare tests only parents' income and work. Four round 2 references paid Tax-Free Childcare to families whose parents fail those tests. to_pe_uk_situation now states is_parent for the adults whenever the household has children. Co-Authored-By: Claude Opus 5.5 --- policybench/scenarios.py | 10 +++++--- tests/test_scenarios.py | 10 +++++++- tests/test_uk_audit.py | 54 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/policybench/scenarios.py b/policybench/scenarios.py index 62a2c65c..35e88d00 100644 --- a/policybench/scenarios.py +++ b/policybench/scenarios.py @@ -675,14 +675,18 @@ def to_pe_uk_situation(self, prefix: str = "") -> dict: f"{self.id} {person.name}: inputs the prompt does not show: " f"{hidden}" ) + is_adult = person.name.startswith("adult") person_data = { "age": self._yearize(person.age), # The stored wage leaf; employment_income adds the (zero) # labour-supply response to it, as in the microsimulation. "employment_income_before_lsr": self._yearize(person.employment_income), - "is_claimant_or_partner": self._yearize( - person.name.startswith("adult") - ), + "is_claimant_or_partner": self._yearize(is_adult), + # The prompt says the children are the adults' dependents. + # PE-UK reads is_parent as an input that defaults to false, + # and Tax-Free Childcare tests only parents' income, so the + # adults are stated as parents whenever there are children. + "is_parent": self._yearize(is_adult and bool(self.children)), } for key, value in person.inputs.items(): if key == "date_of_birth": diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index d0310257..f405d28f 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -1464,9 +1464,16 @@ def test_uk_situation_holds_exactly_the_prompted_facts( people = situation["people"] assert set(people) == {f"s__{p.name}" for p in scenario.all_people} adult = people["s__adult1"] - structural = {"age", "employment_income_before_lsr", "is_claimant_or_partner"} + structural = { + "age", + "employment_income_before_lsr", + "is_claimant_or_partner", + "is_parent", + } assert set(adult) == structural | set(adult_inputs) assert adult["is_claimant_or_partner"] == {"2026": True} + # The prompt makes the adults the children's parents. + assert adult["is_parent"] == {"2026": num_children > 0} assert adult["employment_income_before_lsr"] == {"2026": wage} adult_lines = describe_person(scenario.adults[0], country="uk") for key, value in adult_inputs.items(): @@ -1480,6 +1487,7 @@ def test_uk_situation_holds_exactly_the_prompted_facts( for i in range(num_children): child = people[f"s__child{i + 1}"] assert child["is_claimant_or_partner"] == {"2026": False} + assert child["is_parent"] == {"2026": False} assert child[scenarios_module.UK_EDUCATION_ENTRY_FIELD] == { "2026": scenarios_module.UK_EDUCATION_ENTRY_AGE } diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py index 22ca51ba..41ed0a73 100644 --- a/tests/test_uk_audit.py +++ b/tests/test_uk_audit.py @@ -477,3 +477,57 @@ def test_a_moved_baby_passes_policyengine_uks_age_check(): ) sim = Simulation(situation=scenario.to_pe_uk_situation()) assert float(sim.calculate("child_benefit", 2026).sum()) > 0 + + +def test_adults_with_children_are_stated_as_parents(): + def scenario(children): + return Scenario( + id="scenario_000", + state="LONDON", + filing_status=None, + adults=[ + Person(name="adult1", age=35, employment_income=20_000), + Person(name="adult2", age=34, employment_income=0), + ], + children=children, + country="uk", + ) + + with_child = scenario([Person(name="child1", age=3, employment_income=0)]) + people = with_child.to_pe_uk_situation()["people"] + assert people["adult1"]["is_parent"] == {"2026": True} + assert people["adult2"]["is_parent"] == {"2026": True} + assert people["child1"]["is_parent"] == {"2026": False} + people = scenario([]).to_pe_uk_situation()["people"] + assert people["adult1"]["is_parent"] == {"2026": False} + + +def test_a_parent_who_does_not_work_bars_tax_free_childcare(): + """Tax-Free Childcare needs each parent in qualifying work; the adults + must be parents for the engine to test it.""" + pytest.importorskip("policyengine_uk") + from policyengine_uk import Simulation + + scenario = Scenario( + id="scenario_000", + state="LONDON", + filing_status=None, + adults=[ + Person(name="adult1", age=35, employment_income=30_000), + Person(name="adult2", age=34, employment_income=0), + ], + children=[ + Person( + name="child1", + age=3, + employment_income=0, + inputs={ + "childcare_expenses": 8_000.0, + "tax_free_childcare_uses_qualifying_provider": True, + }, + ) + ], + country="uk", + ) + sim = Simulation(situation=scenario.to_pe_uk_situation()) + assert float(sim.calculate("tax_free_childcare", 2026).sum()) == 0 From 076ef83580044285e4319f35d1890576e2dfaf5a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:29:40 -0400 Subject: [PATCH 8/8] Address the #215 review: keep the audit set out of app data, harden the sampler - export-full-run refuses to write a non-headline output set to the public app data unless --skip-app-data is given. - The sampler counts a missing capital value as nil, never draws from zero-weight candidates, and does not draw a State Pension for an adult whose computed one is already stated. - Note in draw_state_pension that the scheme follows the men's cut-off. - numpy>=1.25 for Generator.spawn; a council-band comment now matches the code. Regenerating with seed 20261011 reproduces all 200 frozen round 2 scenarios exactly. Co-Authored-By: Claude Opus 5.5 --- policybench/full_run_export.py | 8 ++++- policybench/uk_audit.py | 30 ++++++++++++++----- pyproject.toml | 2 +- tests/test_uk_audit.py | 53 ++++++++++++++++++++++++++++++++++ uv.lock | 2 +- 5 files changed, 84 insertions(+), 11 deletions(-) diff --git a/policybench/full_run_export.py b/policybench/full_run_export.py index d993af7b..6b84abc5 100644 --- a/policybench/full_run_export.py +++ b/policybench/full_run_export.py @@ -29,7 +29,7 @@ load_reference_exclusions, split_reference, ) -from policybench.spec import get_output_ids, output_group_id +from policybench.spec import DEFAULT_PROGRAM_SET, get_output_ids, output_group_id class ReferenceProvenanceError(ValueError): @@ -605,6 +605,12 @@ def export_full_run( (the committed pin per country), or a bare sha256 for a single country (see ``reference_digest_for_country``). """ + if program_set not in (None, DEFAULT_PROGRAM_SET) and not skip_app_data: + raise ValueError( + f"Output set {program_set!r} is internal; export it with " + "skip_app_data (--skip-app-data) so it never reaches the public " + "app data." + ) run_path = Path(run_dir) if countries: selected_countries = list(countries) diff --git a/policybench/uk_audit.py b/policybench/uk_audit.py index 6b57ff1a..34d2200a 100644 --- a/policybench/uk_audit.py +++ b/policybench/uk_audit.py @@ -233,14 +233,19 @@ def household_features( float(v) for v in adults[income_columns].sum(axis=1) ), capital=sum( - float(getattr(row, column, 0.0) or 0.0) - for column in CAPITAL_COLUMNS + _amount(getattr(row, column, 0.0)) for column in CAPITAL_COLUMNS ), ) ) return features +def _amount(value) -> float: + """A money amount, with a missing (None or NaN) value as nil.""" + number = pd.to_numeric(value, errors="coerce") + return 0.0 if pd.isna(number) else float(number) + + Selector = Callable[[Household], bool] Overlay = Callable[[Scenario, Household, np.random.Generator], Scenario] @@ -469,6 +474,11 @@ def draw_state_pension(person: Person, rng: np.random.Generator) -> float: of £5-£60 a week. Basic State Pension: 25% the full basic rate, 60% the full basic rate plus £10-£120 a week of additional State Pension, 15% a partial basic rate of 60-99%. + + The scheme is chosen by the men's cut-off (born on or after 6 April 1951). + Women born from 6 April 1951 to 5 April 1953 reached pensionable age + before April 2016 and are on the basic scheme in law. The draw only sets + a plausible amount: the reference reads the amount stated. """ dob = _date_of_birth(person) on_new_scheme = dob is not None and dob >= NEW_STATE_PENSION_FIRST_BIRTH_DATE @@ -505,6 +515,7 @@ def state_pensions(scenario: Scenario, rng: np.random.Generator) -> Scenario: ) if adult.age >= PENSIONER_MIN_AGE and not adult.inputs.get("state_pension_reported") + and not adult.inputs.get("state_pension") else adult for adult in scenario.adults ] @@ -800,8 +811,9 @@ def _scenario_for( if dataset_year is not None: metadata["dataset_year"] = int(dataset_year) household_inputs = _extract_uk_household_inputs(household_row) - # The transfer data's council tax comes from US property tax and its - # bands from US home values; the sample states its own bill instead. + # The transfer data's council tax comes from US property tax, so the + # sample states its own bill (Band D x the band ratio). The band itself is + # reused from the transfer data through metadata, where its nation has it. household_inputs.pop("council_tax", None) household_inputs.pop("council_tax_band", None) return Scenario( @@ -821,13 +833,15 @@ def _scenario_for( def _weighted_draw( candidates: list[Household], n: int, rng: np.random.Generator, name: str = "" ) -> list[Household]: + # A household with no weight can never be drawn, so it is not a candidate. + candidates = [c for c in candidates if c.weight > 0] if len(candidates) < n: raise ValueError( - f"Stratum {name!r} has {len(candidates)} candidate households for " - f"{n} draws." + f"Stratum {name!r} has {len(candidates)} candidate households with " + f"positive weight for {n} draws." ) - weights = np.array([max(c.weight, 0.0) for c in candidates], dtype=float) - p = weights / weights.sum() if weights.sum() > 0 else None + weights = np.array([c.weight for c in candidates], dtype=float) + p = weights / weights.sum() picks = rng.choice(len(candidates), size=n, replace=False, p=p) return [candidates[int(i)] for i in picks] diff --git a/pyproject.toml b/pyproject.toml index 8622b327..89ad6846 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ requires-python = ">=3.11" dependencies = [ "litellm[caching]>=1.30", "pandas>=2.0", - "numpy>=1.24", + "numpy>=1.25", # Generator.spawn (policybench.uk_audit) "policyengine==6.1.2", "policyengine-us==2.38.6", "policyengine-uk==2.126.7", diff --git a/tests/test_uk_audit.py b/tests/test_uk_audit.py index 41ed0a73..83bbf9a0 100644 --- a/tests/test_uk_audit.py +++ b/tests/test_uk_audit.py @@ -531,3 +531,56 @@ def test_a_parent_who_does_not_work_bars_tax_free_childcare(): ) sim = Simulation(situation=scenario.to_pe_uk_situation()) assert float(sim.calculate("tax_free_childcare", 2026).sum()) == 0 + + +def _household(household_id: int, weight: float) -> uk_audit.Household: + return uk_audit.Household( + household_id=household_id, + region="LONDON", + tenure="RENT_PRIVATELY", + weight=weight, + adult_ages=(40,), + child_ages=(), + adult_earnings=(20_000.0,), + adult_self_employment=(0.0,), + adult_income=(20_000.0,), + capital=0.0, + ) + + +def test_weighted_draw_never_counts_zero_weight_households(): + rng = np.random.default_rng(0) + candidates = [_household(1, 1.0), _household(2, 0.0), _household(3, 2.0)] + drawn = uk_audit._weighted_draw(candidates, 2, rng, "test") + assert {h.household_id for h in drawn} == {1, 3} + with pytest.raises(ValueError, match="positive weight"): + uk_audit._weighted_draw(candidates, 3, np.random.default_rng(0), "test") + + +def test_missing_capital_values_count_as_nil(): + assert uk_audit._amount(None) == 0.0 + assert uk_audit._amount(float("nan")) == 0.0 + assert uk_audit._amount("1250.5") == 1250.5 + + +def test_a_stated_state_pension_is_not_drawn_again(): + adult = Person( + name="adult1", age=70, employment_income=0.0, inputs={"state_pension": 9_000.0} + ) + scenario = Scenario( + id="s", + country="uk", + state="LONDON", + filing_status=None, + adults=[adult], + children=[], + ) + out = uk_audit.state_pensions(scenario, np.random.default_rng(0)) + assert "state_pension_reported" not in out.adults[0].inputs + + +def test_an_internal_output_set_never_reaches_the_app_data(tmp_path): + from policybench.full_run_export import export_full_run + + with pytest.raises(ValueError, match="internal"): + export_full_run(run_dir=tmp_path, program_set="uk_audit", skip_app_data=False) diff --git a/uv.lock b/uv.lock index 5baed6ed..95427d91 100644 --- a/uv.lock +++ b/uv.lock @@ -2507,7 +2507,7 @@ requires-dist = [ { name = "litellm", extras = ["caching"], specifier = ">=1.30" }, { name = "nbclient", marker = "extra == 'docs'", specifier = ">=0.10" }, { name = "nbformat", marker = "extra == 'docs'", specifier = ">=5.0" }, - { name = "numpy", specifier = ">=1.24" }, + { name = "numpy", specifier = ">=1.25" }, { name = "pandas", specifier = ">=2.0" }, { name = "policyengine", specifier = "==6.1.2" }, { name = "policyengine-uk", specifier = "==2.126.7" },