Skip to content

Add a stratified UK sample and audit outputs for round 2 of the policyengine-uk audit #215

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MaxGhenis merged 8 commits into from
Oct 11, 2026
Merged

Add a stratified UK sample and audit outputs for round 2 of the policyengine-uk audit #215

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d1063ff
Add a stratified UK sample and audit outputs for round 2 of the polic…
MaxGhenis Oct 11, 2026
6b4ff53
Select the low-income strata on capital, not only savings
MaxGhenis Oct 11, 2026
24f84f5
Let the full-run export keep a named output set; fix a json shadowing…
MaxGhenis Oct 11, 2026
21a0ae6
Keep UK audit birth dates before the fiscal year; print UK losses as -£
MaxGhenis Oct 11, 2026
85a34ed
Keep the transfer path stating one State Pension; update UK test expe…
MaxGhenis Oct 11, 2026
95caec8
Move a baby born during the fiscal year to a date that keeps it aged 0
MaxGhenis Oct 11, 2026
c216384
State the adults of a UK household with children as parents
MaxGhenis Oct 11, 2026
076ef83
Address the #215 review: keep the audit set out of app data, harden t…
MaxGhenis Oct 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 161 additions & 0 deletions policybench/benchmark_specs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,167 @@
}
]
}
},
"uk_audit": {
"description": "Internal policyengine-uk audit (round 2, October 2026): the UK headline outputs plus the High Income Child Benefit Charge, Carer's Allowance, Attendance Allowance, Housing Benefit, Council Tax Reduction, Tax-Free Childcare and Scottish Child Payment. Not a public board.",
"countries": {
"uk": [
{
"id": "income_tax",
"pe_variable": "income_tax",
"label": "Income Tax",
"prompt": "household total annual UK Income Tax liability after allowances and reliefs, excluding Capital Gains Tax",
"metric_type": "amount",
"role": "tax",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": -1
},
{
"id": "national_insurance",
"pe_variable": "national_insurance",
"label": "National Insurance",
"prompt": "household total annual UK National Insurance contributions, excluding employer National Insurance",
"metric_type": "amount",
"role": "tax",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": -1
},
{
"id": "capital_gains_tax",
"pe_variable": "capital_gains_tax",
"label": "Capital Gains Tax",
"prompt": "household total annual UK Capital Gains Tax. Compute it separately from Income Tax. UK has no short-term versus long-term capital gains distinction; treat any listed capital gains as ordinary net chargeable gains before the annual exempt amount, excluding carried interest and special reliefs unless explicitly stated. Sum across household members",
"metric_type": "amount",
"role": "tax",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": -1
},
{
"id": "child_benefit",
"pe_variable": "child_benefit",
"label": "Child Benefit",
"prompt": "household total annual gross Child Benefit amount for qualifying children and young people before the High Income Child Benefit Charge; assume eligible households take up the benefit and do not require stated benefit receipt; do not apply an income test or tax-charge reduction to this output; do not subtract HICBC here because it is included in Income Tax, so report gross Child Benefit even when HICBC would recover it through tax",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "universal_credit",
"pe_variable": "universal_credit",
"label": "Universal Credit",
"prompt": "household total annual Universal Credit amount; assume eligible households take up the benefit and do not require stated benefit receipt",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "pension_credit",
"pe_variable": "pension_credit",
"label": "Pension Credit",
"prompt": "annual Pension Credit amount",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "pip",
"pe_variable": "pip",
"label": "Personal Independence Payment",
"prompt": "annual Personal Independence Payment (PIP) amount",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "high_income_child_benefit_charge",
"pe_variable": "CB_HITC",
"label": "High Income Child Benefit Charge",
"prompt": "household total annual High Income Child Benefit Charge on the Child Benefit the household receives (0 if none). The Income Tax output includes this charge; report the charge here on its own as well",
"metric_type": "amount",
"role": "tax",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": -1
},
{
"id": "carers_allowance",
"pe_variable": "carers_allowance",
"label": "Carer's Allowance",
"prompt": "household total annual Carer's Allowance payable, after any reduction for an overlapping benefit such as the State Pension; assume anyone entitled claims it",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "attendance_allowance",
"pe_variable": "attendance_allowance",
"label": "Attendance Allowance",
"prompt": "household total annual Attendance Allowance",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "housing_benefit",
"pe_variable": "housing_benefit",
"label": "Housing Benefit",
"prompt": "household total annual Housing Benefit; assume anyone entitled claims it",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "council_tax_reduction",
"pe_variable": "council_tax_reduction",
"label": "Council Tax Reduction",
"prompt": "household total annual Council Tax Reduction (council tax support) under the scheme of the household's council, as a reduction in the stated Council Tax bill; assume anyone entitled claims it",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "tax_free_childcare",
"pe_variable": "tax_free_childcare",
"label": "Tax-Free Childcare",
"prompt": "household total annual government top-up paid through Tax-Free Childcare for the stated childcare costs; assume the household claims every benefit it is entitled to, including Universal Credit, and that an eligible family opens an account and pays all the stated childcare costs through it",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
},
{
"id": "scottish_child_payment",
"pe_variable": "scottish_child_payment",
"label": "Scottish Child Payment",
"prompt": "household total annual Scottish Child Payment; assume anyone entitled claims it",
"metric_type": "amount",
"role": "benefit",
"output_set": "headline",
"aggregation": "sum",
"net_income_sign": 1
}
]
}
}
}
}
32 changes: 32 additions & 0 deletions policybench/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,11 @@ def main():
action="store_true",
help="Only write the combined payload under the run directory",
)
export_parser.add_argument(
"--program-set",
default=None,
help="Output set the run asked for (default: headline)",
)

# Failure audit (Codex-backed classifier)
audit_prepare_parser = subparsers.add_parser(
Expand Down Expand Up @@ -1019,6 +1024,11 @@ def main():
help="Stop dispatching at 90%% of this spend; omit for no cap",
)
run_parser.add_argument("--max-workers", type=int, default=4)
run_parser.add_argument(
"--program-set",
default=None,
help="Benchmark output set the workers request (default: headline)",
)
run_parser.add_argument(
"--max-rounds",
type=int,
Expand All @@ -1039,6 +1049,16 @@ def main():
"--report-output", default=None, help="Optional path for the report"
)

uk_audit_parser = subparsers.add_parser(
"uk-audit-sample",
help="Draw the stratified UK household sample for the policyengine-uk "
"audit and write its scenario manifest",
)
uk_audit_parser.add_argument(
"-o", "--output", default="results/local/uk_audit/scenarios.csv"
)
uk_audit_parser.add_argument("--seed", type=int, default=None)

fold_parser = subparsers.add_parser(
"fold-board",
help="Fold new model prediction CSVs into an existing board and "
Expand Down Expand Up @@ -1090,6 +1110,16 @@ def main():

enable_cache()

if args.command == "uk-audit-sample":
from policybench.uk_audit import UK_AUDIT_SEED, write_uk_audit_sample

meta = write_uk_audit_sample(
args.output, seed=UK_AUDIT_SEED if args.seed is None else args.seed
)
print(json.dumps(meta["counts"], indent=1))
print(f"UK audit sample saved to {args.output}")
return

if args.command == "reference-outputs":
from policybench.ground_truth import calculate_ground_truth
from policybench.policyengine_runtime import runtime_metadata_for_country
Expand Down Expand Up @@ -1197,6 +1227,7 @@ def main():
budget_usd=args.budget_usd,
max_workers=args.max_workers,
max_rounds=args.max_rounds,
program_set=args.program_set,
)
state = supervisor.run()
except ValueError as exc:
Expand Down Expand Up @@ -1375,6 +1406,7 @@ def main():
reference_digest=parse_reference_digest_args(args.reference_digest),
app_data_output=args.app_data_output,
skip_app_data=args.skip_app_data,
program_set=args.program_set,
)
except (FileNotFoundError, ReferenceProvenanceError) as exc:
raise SystemExit(str(exc)) from exc
Expand Down
34 changes: 25 additions & 9 deletions policybench/full_run_export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
load_reference_exclusions,
split_reference,
)
from policybench.spec import get_output_ids, output_group_id
from policybench.spec import DEFAULT_PROGRAM_SET, get_output_ids, output_group_id


class ReferenceProvenanceError(ValueError):
Expand All @@ -46,15 +46,18 @@ class ReferenceProvenanceError(ValueError):
)


def _canonical_output_ids(country: str) -> set[str]:
return set(get_output_ids(country, "headline"))
def _canonical_output_ids(country: str, program_set: str | None = None) -> set[str]:
return set(get_output_ids(country, program_set or "headline"))


def _filter_to_canonical_outputs(frame: pd.DataFrame, country: str) -> pd.DataFrame:
"""Drop source-run rows outside the canonical headline output scope."""
def _filter_to_canonical_outputs(
frame: pd.DataFrame, country: str, program_set: str | None = None
) -> pd.DataFrame:
"""Drop source-run rows outside the run's output set (the headline set
unless the run names another, such as the internal ``uk_audit`` set)."""
if frame.empty or "variable" not in frame.columns:
return frame
canonical = _canonical_output_ids(country)
canonical = _canonical_output_ids(country, program_set)
return frame[frame["variable"].map(output_group_id).isin(canonical)].reset_index(
drop=True
)
Expand Down Expand Up @@ -405,7 +408,12 @@ def reference_digest_for_country(
return reference_digest


def export_country(country_dir: Path, *, reference_digest: str | None = None) -> dict:
def export_country(
country_dir: Path,
*,
reference_digest: str | None = None,
program_set: str | None = None,
) -> dict:
"""Write analysis artifacts and dashboard payload for one country run.

Reference provenance is verified strictly: the sidecar must carry the
Expand Down Expand Up @@ -446,8 +454,8 @@ def export_country(country_dir: Path, *, reference_digest: str | None = None) ->
predictions,
load_case_reference_explanations(country_dir),
)
ground_truth = _filter_to_canonical_outputs(ground_truth, country)
predictions = _filter_to_canonical_outputs(predictions, country)
ground_truth = _filter_to_canonical_outputs(ground_truth, country, program_set)
predictions = _filter_to_canonical_outputs(predictions, country, program_set)

# Outputs whose reference depends on an input the data never carried are
# scored for no model; they stay in the payload marked scored=false.
Expand Down Expand Up @@ -588,6 +596,7 @@ def export_full_run(
app_data_output: str | Path = "app/src/data.json",
skip_app_data: bool = False,
reference_digest: ReferenceDigests = None,
program_set: str | None = None,
) -> dict:
"""Export per-country and combined frontend artifacts from a full run.

Expand All @@ -596,6 +605,12 @@ def export_full_run(
(the committed pin per country), or a bare sha256 for a single country
(see ``reference_digest_for_country``).
"""
if program_set not in (None, DEFAULT_PROGRAM_SET) and not skip_app_data:
raise ValueError(
f"Output set {program_set!r} is internal; export it with "
"skip_app_data (--skip-app-data) so it never reaches the public "
"app data."
)
run_path = Path(run_dir)
if countries:
selected_countries = list(countries)
Expand All @@ -614,6 +629,7 @@ def export_full_run(
reference_digest=reference_digest_for_country(
reference_digest, country, len(selected_countries)
),
program_set=program_set,
)
for country in selected_countries
}
Expand Down
Loading
Loading