From 1e980719ee3e30b5ffefc627eae0cbf48c4776e2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 13:43:15 -0400 Subject: [PATCH 01/10] Add an aggregate-impact report and pin the certified test dataset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report measures one model checkout on the pinned certified dataset over 2025-26 to 2031-32: receipts by tax head, spending and caseloads by benefit, poverty by measure and group, inequality, own-decile incomes and the standard reform costings. It checks them against a fiscal partition of the model's totals (policyengine_uk/data/fiscal_partition.yaml): - closure: each total's components are exactly the partition's heads and declared double counts; - accounting identities, household by household; - a counted-once check, which perturbs each component by £1 and requires each tax total to move by exactly £1 per unit, and each spending total by at most £1. compare then sets a base measurement against a head one. It fails on a crash, a different dataset, non-finite figures or a check the base passes and the head breaks. It flags moves above the proposed thresholds, and renders a pull request comment. The partition's tax heads follow the #2159 work's specification, encoded for main before that fix: corporate_sdlt is a declared double count inside expected_sdlt. Spending heads cover gov_spending and household_benefits. Test data: conftest pins enhanced_frs_2024_25 from uk-data 1.58.0 by Hugging Face commit and SHA-256 (tests/certified_dataset.yaml), verifies the bytes before data tests run, and adds a mandatory-data mode (POLICYENGINE_UK_REQUIRE_TEST_DATA). In that mode, data tests fail rather than skip without data, and run only on the pinned dataset. The reform-cost fixtures are rebased onto 1.58.0, keeping each earlier value as a receipt with the model and data that produced it. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2334-test-data.changed.md | 1 + changelog.d/2334.added.md | 1 + policyengine_uk/data/fiscal_partition.yaml | 951 ++++++++++++++++++ .../test_aggregate_impact_certified.py | 144 +++ .../test_aggregate_impact_compare.py | 311 ++++++ .../test_aggregate_impact_dataset.py | 191 ++++ .../test_aggregate_impact_guard.py | 212 ++++ .../test_aggregate_impact_partition.py | 203 ++++ policyengine_uk/tests/certified_dataset.yaml | 20 + policyengine_uk/tests/conftest.py | 175 +++- .../tests/microsimulation/reforms_config.yaml | 113 ++- .../utils/aggregate_impact/__init__.py | 14 + .../utils/aggregate_impact/__main__.py | 196 ++++ policyengine_uk/utils/aggregate_impact/api.py | 145 +++ .../utils/aggregate_impact/compare.py | 376 +++++++ .../utils/aggregate_impact/dataset.py | 164 +++ .../utils/aggregate_impact/guard.py | 276 +++++ .../utils/aggregate_impact/measure.py | 765 ++++++++++++++ .../utils/aggregate_impact/partition.py | 545 ++++++++++ .../utils/aggregate_impact/render.py | 375 +++++++ .../utils/aggregate_impact/report_spec.yaml | 88 ++ .../utils/aggregate_impact/spec.py | 105 ++ 22 files changed, 5346 insertions(+), 25 deletions(-) create mode 100644 changelog.d/2334-test-data.changed.md create mode 100644 changelog.d/2334.added.md create mode 100644 policyengine_uk/data/fiscal_partition.yaml create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py create mode 100644 policyengine_uk/tests/certified_dataset.yaml create mode 100644 policyengine_uk/utils/aggregate_impact/__init__.py create mode 100644 policyengine_uk/utils/aggregate_impact/__main__.py create mode 100644 policyengine_uk/utils/aggregate_impact/api.py create mode 100644 policyengine_uk/utils/aggregate_impact/compare.py create mode 100644 policyengine_uk/utils/aggregate_impact/dataset.py create mode 100644 policyengine_uk/utils/aggregate_impact/guard.py create mode 100644 policyengine_uk/utils/aggregate_impact/measure.py create mode 100644 policyengine_uk/utils/aggregate_impact/partition.py create mode 100644 policyengine_uk/utils/aggregate_impact/render.py create mode 100644 policyengine_uk/utils/aggregate_impact/report_spec.yaml create mode 100644 policyengine_uk/utils/aggregate_impact/spec.py diff --git a/changelog.d/2334-test-data.changed.md b/changelog.d/2334-test-data.changed.md new file mode 100644 index 0000000000..c82ac32d4d --- /dev/null +++ b/changelog.d/2334-test-data.changed.md @@ -0,0 +1 @@ +- Microsimulation tests now run on the certified uk-data 1.58.0 dataset, pinned by commit and SHA-256, and fail instead of skipping when `POLICYENGINE_UK_REQUIRE_TEST_DATA` is set and the data is unavailable; the reform-cost fixtures are rebased onto it, keeping their earlier values as receipts. diff --git a/changelog.d/2334.added.md b/changelog.d/2334.added.md new file mode 100644 index 0000000000..c58137024c --- /dev/null +++ b/changelog.d/2334.added.md @@ -0,0 +1 @@ +- Added an aggregate-impact report (`python -m policyengine_uk.utils.aggregate_impact`) that measures a model checkout on the pinned certified dataset and compares base with head: receipts by tax head, spending and caseloads, poverty, inequality, deciles and the standard reform costings over 2025-26 to 2031-32, with accounting identities and a counted-once check against a reviewed fiscal partition (`policyengine_uk/data/fiscal_partition.yaml`). diff --git a/policyengine_uk/data/fiscal_partition.yaml b/policyengine_uk/data/fiscal_partition.yaml new file mode 100644 index 0000000000..bf03754400 --- /dev/null +++ b/policyengine_uk/data/fiscal_partition.yaml @@ -0,0 +1,951 @@ +# Fiscal partition: how the model's tax and spending totals divide into heads. +# +# Each head is one receipt (an OBR current-receipts line, split by who pays +# where the model splits it) and names the one variable that measures the +# household's share of it. Heads are mutually exclusive: no head's value +# contains another head's value, and no variable is summed twice in a total. +# +# The heads were specified from the OBR receipts classification and from each +# variable's formula, not copied from the model's own lists +# (GOV_TAX_VARIABLES, HOUSEHOLD_TAX_VARIABLES, +# PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES). policyengine_uk/tests/ +# test_fiscal_partition.py checks the two against each other and checks the +# accounting properties below on simulated households. +# +# Spending heads divide gov_spending and household_benefits the same way: +# one benefit or service per head, one variable each. +# +# This version describes main before the #2159 fix: expected_sdlt still adds +# corporate_sdlt, so the household SDLT head carries corporate SDLT too and +# corporate_sdlt is declared under model_double_counts. The fix gives +# corporate_sdlt its own head (sdlt_corporate) and empties that section. +# +# Consumers (tests, the per-PR aggregate impact report) read this file +# directly. Bump schema_version on any change to field meanings. +# +# Fields +# tax_heads[]: +# id one per head +# label sentence-case name +# variable the one model variable that measures the head +# entity household or person +# obr_line OBR current-receipts line the head belongs to, or +# not_an_obr_current_receipt +# report_group grouping for reports +# payer who bears it in the model: household (the household +# or its members pay it directly), employer, consumer +# incidence (allocated by consumption), corporate +# incidence (allocated by shareholding), or mixed +# totals the totals that include it +# sign nonnegative (at current-law parameter values, for +# non-negative inputs) or any (a change measure) +# zero_at_baseline expected: current-law parameters make it zero for +# every household (a reform-only head) +# reads other heads this head's formula reaches, with the +# route; reads that change eligibility, not inclusion +# exclusions[]: variables that must never be summed alongside the heads. +# kind: component (summed or read inside `head`), measure_within (an +# analytical decomposition already inside `head` through its inputs), +# composite (spans several heads), duplicate (same value as `head`), +# baseline_measure. +# model_double_counts[]: known violations still in the model: a variable a +# total lists although the head variable `contained_in` already adds it. +# spending_heads[]: as tax_heads, without obr_line and payer, plus +# caseload (default true): whether reports count recipients. +# overlap_guards[]: the places double counting could occur, and the +# property the tests check for each. +schema_version: 1 + +totals: + gov_tax: + label: Government tax revenue in respect of the household + description: Receipts at their level, including employer-paid taxes and VAT. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + - high_value_council_tax_surcharge + household_tax: + label: Taxes borne by the household + description: >- + Taxes subtracted from household net income. VAT and employer NI enter + only as reform-induced changes (vat_change and the employer NI response + incidence), because baseline VAT and employer NI are not deducted from + the incomes the model starts from. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + - high_value_council_tax_surcharge + pre_budget_change_household_tax: + label: Taxes borne by the household, pre-budget-change definition + description: >- + A fixed earlier definition of household_tax used by + pre_budget_change_household_net_income. It omits heads added since + (alcohol, tobacco and vehicle excise duties, private school VAT, the + high value council tax surcharge, student loan repayments, and the + incidence-change heads). + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + + gov_spending: + label: Government spending in respect of the household + description: >- + Cash benefits, childcare and student support, and public services in + kind (NHS, education, transport subsidies). + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax_benefit + household_benefits: + label: Benefits received by the household + description: >- + The benefit side of household net income. It omits pawhp and the + contributed budget-change line, and adds Scottish Child Payment, Healthy + Start and the Two Child Limit Payment, which gov_spending leaves out. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax_benefit + +tax_heads: + # Income and earnings + - id: income_tax + label: Income tax + variable: income_tax + entity: person + obr_line: Income tax + report_group: income_tax + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: nics_individual + label: National Insurance (employee and self-employed) + variable: national_insurance + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: nics_employer + label: National Insurance (employer) + variable: ni_employer + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: employer + totals: [gov_tax] + sign: nonnegative + - id: employer_nics_response_capital + label: Employer NI change borne by capital + variable: employer_ni_response_capital_incidence + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: corporate incidence + totals: [household_tax] + sign: any + zero_at_baseline: expected + - id: employer_nics_response_consumer + label: Employer NI change borne by consumers + variable: employer_ni_response_consumer_incidence + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: consumer incidence + totals: [household_tax] + sign: any + zero_at_baseline: expected + - id: capital_gains_tax + label: Capital gains tax + variable: capital_gains_tax + entity: person + obr_line: Capital gains tax + report_group: capital_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: student_loan_repayments + label: Student loan repayments + variable: student_loan_repayments + entity: person + obr_line: not_an_obr_current_receipt + obr_line_note: >- + A financial transaction: repayments reduce the student loan book, not + borrowing, so they are outside current receipts. The model counts them + in gov_tax and household_tax; reports should show them separately. + report_group: financial_transactions + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + + # Consumption + - id: vat + label: VAT + variable: vat + entity: household + obr_line: Value added tax + report_group: vat + payer: consumer incidence + totals: [gov_tax] + sign: nonnegative + - id: vat_change + label: Change in VAT from its baseline + variable: vat_change + entity: household + obr_line: Value added tax + report_group: vat + payer: consumer incidence + totals: [household_tax, pre_budget_change_household_tax] + sign: any + zero_at_baseline: expected + reads: + - head: vat + route: vat_change adds vat and subtracts baseline_vat + why: a change measure built from the level; the two are never in the same total + - id: private_school_vat + label: VAT on private school fees + variable: private_school_vat + entity: household + obr_line: Value added tax + report_group: vat + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + zero_at_baseline: expected + zero_at_baseline_note: >- + gov.contrib.labour.private_school_vat is 0 at current law in this + model, so the head is reform-only here. + reads: + - head: income_tax + route: private_school_vat -> attends_private_school -> income_tax + why: attendance is imputed from income; it decides who attends, not how much tax is counted + - head: nics_individual + route: private_school_vat -> attends_private_school -> (net income) -> national_insurance + why: as above + - head: council_tax + route: private_school_vat -> attends_private_school -> household_benefits -> council_tax_benefit -> council_tax + why: as above + - id: fuel_duty + label: Fuel duties + variable: fuel_duty + entity: household + obr_line: Fuel duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: alcohol_duty + label: Alcohol duties + variable: alcohol_duty + entity: household + obr_line: Alcohol duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: tobacco_duty + label: Tobacco duties + variable: tobacco_duty + entity: household + obr_line: Tobacco duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: vehicle_excise_duty + label: Vehicle excise duties + variable: vehicle_excise_duty + entity: household + obr_line: Vehicle excise duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: tv_licence + label: TV licence fee + variable: tv_licence + entity: household + obr_line: Licence fee receipts + report_group: other_receipts + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + reads: + - head: income_tax + route: tv_licence -> tv_licence_discount -> pension_credit -> pension_credit_income -> income_tax + why: the over-75 discount depends on Pension Credit receipt, a means test on net income + - head: nics_individual + route: tv_licence -> tv_licence_discount -> pension_credit -> pension_credit_income -> national_insurance + why: as above + + # Property + - id: council_tax + label: Council tax + variable: council_tax + entity: household + obr_line: Council tax + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: high_value_council_tax_surcharge + label: High value council tax surcharge + variable: high_value_council_tax_surcharge + entity: household + obr_line: Council tax + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + - id: domestic_rates + label: Northern Ireland domestic rates + variable: domestic_rates + entity: household + obr_line: Council tax + obr_line_note: >- + Northern Ireland's domestic rates are its counterpart to council tax; + check the line against the OBR table in use before publishing a + line-by-line comparison. + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: business_rates + label: Business rates + variable: business_rates + entity: household + obr_line: Business rates + report_group: business_rates + payer: corporate incidence + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: sdlt_household + label: Stamp duty land tax (own purchases, and corporate until the #2159 fix) + variable: expected_sdlt + entity: household + obr_line: Stamp duty land tax + report_group: property_transaction_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + Before the #2159 fix expected_sdlt adds corporate_sdlt and a stray 0.045 + per household, so this head is both parts of the receipt. + - id: land_transaction_tax + label: Land transaction tax (Wales) + variable: expected_ltt + entity: household + obr_line: Land transaction tax + report_group: property_transaction_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + expected_ltt multiplies an already purchase-drawn liability by + property_sale_rate, discounting it twice (issue 2246). + - id: land_and_buildings_transaction_tax + label: Land and buildings transaction tax (Scotland) + variable: expected_lbtt + entity: household + obr_line: Land and buildings transaction tax + report_group: property_transaction_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + expected_lbtt multiplies an already purchase-drawn liability by + property_sale_rate, discounting it twice (issue 2246). + + # Reform-only heads + - id: corporate_incident_tax_change + label: Unspecified tax change borne by capital + variable: corporate_incident_tax_revenue_change + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A lump-sum revenue change allocated by shareholding. + report_group: incidence_changes + payer: corporate incidence + totals: [gov_tax, household_tax] + sign: any + zero_at_baseline: expected + - id: consumer_incident_tax_change + label: Unspecified tax change borne by consumers + variable: consumer_incident_tax_revenue_change + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A lump-sum revenue change allocated by consumption. + report_group: incidence_changes + payer: consumer incidence + totals: [gov_tax, household_tax] + sign: any + zero_at_baseline: expected + - id: wealth_tax + label: Wealth tax + variable: wealth_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: non_primary_residence_wealth_tax + label: Wealth tax excluding the main residence + variable: non_primary_residence_wealth_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: land_value_tax + label: Land value tax + variable: LVT + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: carbon_tax + label: Carbon tax + variable: carbon_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + +exclusions: + # Property transaction taxes + - {variable: stamp_duty_land_tax, kind: component, head: sdlt_household, note: "Equal to expected_sdlt, but not zeroed by gov.hmrc.stamp_duty.abolish; report the head, not this."} + - {variable: sdlt_on_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_rent, kind: component, head: sdlt_household} + - {variable: sdlt_on_residential_property_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_non_residential_property_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_residential_property_rent, kind: component, head: sdlt_household} + - {variable: sdlt_on_non_residential_property_rent, kind: component, head: sdlt_household} + - {variable: land_transaction_tax, kind: component, head: land_transaction_tax, note: "The liability before the property_sale_rate scaling (issue 2246)."} + - {variable: ltt_on_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_rent, kind: component, head: land_transaction_tax} + - {variable: ltt_on_residential_property_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_non_residential_property_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_residential_property_rent, kind: component, head: land_transaction_tax} + - {variable: ltt_on_non_residential_property_rent, kind: component, head: land_transaction_tax} + - {variable: land_and_buildings_transaction_tax, kind: component, head: land_and_buildings_transaction_tax, note: "The liability before the property_sale_rate scaling (issue 2246)."} + - {variable: lbtt_on_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_residential_property_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_non_residential_property_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_residential_property_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_non_residential_property_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: corporate_tax_incidence, kind: composite, spans: [sdlt_household, business_rates]} + - {variable: hbai_excluded_income, kind: composite, spans: [sdlt_household, business_rates], note: "Minus corporate_tax_incidence."} + - {variable: baseline_business_rates, kind: duplicate, head: business_rates, note: "Same formula and current-law parameters as business_rates."} + # Income tax and National Insurance + - {variable: income_tax_pre_charges, kind: component, head: income_tax} + - {variable: earned_income_tax, kind: component, head: income_tax} + - {variable: savings_income_tax, kind: component, head: income_tax} + - {variable: dividend_income_tax, kind: component, head: income_tax} + - {variable: property_income_tax, kind: component, head: income_tax} + - {variable: CB_HITC, kind: component, head: income_tax} + - {variable: personal_pension_contributions_tax, kind: component, head: income_tax} + - {variable: ni_employee, kind: measure_within, head: nics_individual, note: "Class 1 employee; national_insurance adds the classes directly."} + - {variable: ni_self_employed, kind: measure_within, head: nics_individual, note: "Classes 2 and 4; national_insurance adds the classes directly."} + - {variable: ni_class_1_employee, kind: component, head: nics_individual} + - {variable: ni_class_1_employee_primary, kind: component, head: nics_individual} + - {variable: ni_class_1_employee_additional, kind: component, head: nics_individual} + - {variable: ni_class_2, kind: component, head: nics_individual} + - {variable: ni_class_3, kind: component, head: nics_individual} + - {variable: ni_class_4, kind: component, head: nics_individual} + - {variable: ni_class_1_employer, kind: component, head: nics_employer} + - {variable: salary_sacrifice_pension_ni_employee, kind: measure_within, head: nics_individual, note: "The excess sacrifice is added back to employment_income, so Class 1 already charges it."} + - {variable: salary_sacrifice_pension_ni_employer, kind: measure_within, head: nics_employer, note: "As above, for employer Class 1."} + - {variable: tax, kind: composite, spans: [income_tax, nics_individual]} + - {variable: benunit_tax, kind: composite, spans: [income_tax, nics_individual]} + - {variable: total_national_insurance, kind: composite, spans: [nics_individual, nics_employer]} + - {variable: student_loan_repayment, kind: component, head: student_loan_repayments} + # Consumption + - {variable: baseline_vat, kind: baseline_measure, head: vat_change, note: "Subtracted inside vat_change."} + - {variable: beer_duty, kind: component, head: alcohol_duty} + - {variable: cider_duty, kind: component, head: alcohol_duty} + - {variable: sparkling_cider_duty, kind: component, head: alcohol_duty} + - {variable: wine_duty, kind: component, head: alcohol_duty} + - {variable: spirits_duty, kind: component, head: alcohol_duty} + - {variable: other_fermented_duty, kind: component, head: alcohol_duty} + - {variable: car_vehicle_excise_duty, kind: component, head: vehicle_excise_duty} + # Council tax net of a benefit + - {variable: council_tax_less_benefit, kind: composite, spans: [council_tax], note: "Council tax minus Council Tax Reduction, a benefit; not a receipt."} + +model_double_counts: + - variable: corporate_sdlt + contained_in: expected_sdlt + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + issue: https://github.com/PolicyEngine/policyengine-uk/issues/2159 + +overlap_guards: + - id: sdlt_payer_split + kind: payer_split + heads: [sdlt_household] + property: >- + Until the #2159 fix, expected_sdlt adds corporate_sdlt, which the totals + also list (model_double_counts). The fix splits the head in two, neither + reading the other, both zeroed by gov.hmrc.stamp_duty.abolish. + - id: property_transaction_tax_geography + kind: geographic_exclusivity + heads: [sdlt_household, land_transaction_tax, land_and_buildings_transaction_tax] + property: At most one is non-zero for any household (England and Northern Ireland, Wales, Scotland). + - id: local_tax_geography + kind: geographic_exclusivity + heads: [council_tax, domestic_rates] + property: >- + No household has both. Both are inputs, so this is a property of the + data, checked on the certified dataset. + - id: vat_level_and_change + kind: level_and_change + heads: [vat, vat_change] + property: Never in the same total. vat_change reads vat. + - id: employer_nics_level_and_incidence + kind: level_and_change + heads: [nics_employer, employer_nics_response_capital, employer_nics_response_consumer] + property: The level is only in gov_tax; the response incidence is only in household_tax. + - id: corporate_incidence_allocations + kind: shared_allocation_key + heads: [business_rates, sdlt_household, corporate_incident_tax_change, carbon_tax, land_value_tax] + property: >- + Each allocates a different receipt by shareholding; none reads another. + corporate_tax_incidence sums two of them and is excluded. + - id: private_school_vat_and_vat + kind: possible_base_overlap + heads: [vat, private_school_vat] + property: >- + vat applies to imputed consumption; if that consumption includes school + fees, turning on private_school_vat would tax them twice. The head is + zero at baseline, so baseline totals are unaffected; check the + consumption imputation before costing a reform that sets it. + - id: eligibility_reads + kind: read_not_inclusion + heads: [tv_licence, private_school_vat] + property: >- + These heads reach other heads of the same total only through + eligibility or imputation (see each head's reads). Raising the other + head by a small amount leaves them unchanged on the test population. + +spending_heads: + # Pensioners + - id: state_pension + label: State Pension + variable: state_pension + entity: person + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: pension_credit + label: Pension Credit + variable: pension_credit + entity: benunit + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: winter_fuel_payment + label: Winter Fuel Payment + variable: winter_fuel_allowance + entity: household + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: pawhp + label: Pension Age Winter Heating Payment (Scotland) + variable: pawhp + entity: household + report_group: pensioner_benefits + totals: [gov_spending] + sign: nonnegative + model_note: >- + Counted in gov_spending but not in household_benefits or household net + income (PR 2047). + # Working-age means-tested benefits + - id: universal_credit + label: Universal Credit + variable: universal_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: housing_benefit + label: Housing Benefit + variable: housing_benefit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: council_tax_reduction + label: Council Tax Reduction + variable: council_tax_benefit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: esa_income + label: Employment and Support Allowance (income-related) + variable: esa_income + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: esa_contrib + label: Employment and Support Allowance (contribution-based) + variable: esa_contrib + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: jsa_income + label: Jobseeker's Allowance (income-based) + variable: jsa_income + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: jsa_contrib + label: Jobseeker's Allowance (contribution-based) + variable: jsa_contrib + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: income_support + label: Income Support + variable: income_support + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: working_tax_credit + label: Working Tax Credit + variable: working_tax_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: child_tax_credit + label: Child Tax Credit + variable: child_tax_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: incapacity_benefit + label: Incapacity Benefit + variable: incapacity_benefit + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Disability and carers + - id: pip + label: Personal Independence Payment + variable: pip + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: dla + label: Disability Living Allowance + variable: dla + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: attendance_allowance + label: Attendance Allowance + variable: attendance_allowance + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: carers_allowance + label: Carer's Allowance + variable: carers_allowance + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: carer_support_payment + label: Carer Support Payment (Scotland) + variable: carer_support_payment + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: scottish_carer_supplement + label: Carer's Allowance Supplement (Scotland) + variable: scottish_carer_supplement + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: sda + label: Severe Disablement Allowance + variable: sda + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: iidb + label: Industrial Injuries Disablement Benefit + variable: iidb + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: afcs + label: Armed Forces Compensation Scheme + variable: afcs + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Children and families + - id: child_benefit + label: Child Benefit + variable: child_benefit + entity: benunit + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: scottish_child_payment + label: Scottish Child Payment + variable: scottish_child_payment + entity: person + report_group: child_and_family_benefits + totals: [household_benefits] + sign: nonnegative + model_note: >- + Counted in household_benefits but missing from gov_spending, so + gov_balance leaves out its cost (about £0.5bn a year on uk-data 1.58.0). + - id: two_child_limit_payment + label: Two Child Limit Payment (Scotland, counterfactual) + variable: two_child_limit_payment + entity: benunit + report_group: child_and_family_benefits + totals: [household_benefits] + sign: nonnegative + zero_at_baseline: expected + model_note: Counted in household_benefits but not in gov_spending. + - id: healthy_start_vouchers + label: Healthy Start vouchers + variable: healthy_start_vouchers + entity: person + report_group: child_and_family_benefits + totals: [household_benefits] + sign: nonnegative + model_note: Counted in household_benefits but missing from gov_spending. + - id: maternity_allowance + label: Maternity Allowance + variable: maternity_allowance + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_maternity_pay + label: Statutory Maternity Pay + variable: statutory_maternity_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_sick_pay + label: Statutory Sick Pay + variable: statutory_sick_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: ssmg + label: Sure Start Maternity Grant + variable: ssmg + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: bsp + label: Bereavement Support Payment + variable: bsp + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Childcare and education support + - id: tax_free_childcare + label: Tax-Free Childcare + variable: tax_free_childcare + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: extended_childcare_entitlement + label: Extended childcare entitlement + variable: extended_childcare_entitlement + entity: benunit + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: universal_childcare_entitlement + label: Universal childcare entitlement + variable: universal_childcare_entitlement + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: targeted_childcare_entitlement + label: Targeted childcare entitlement + variable: targeted_childcare_entitlement + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: care_to_learn + label: Care to Learn + variable: care_to_learn + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: childcare_grant + label: Childcare Grant + variable: childcare_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: parents_learning_allowance + label: Parents' Learning Allowance + variable: parents_learning_allowance + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: adult_dependants_grant + label: Adult Dependants' Grant + variable: adult_dependants_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: travel_grant + label: Travel Grant + variable: travel_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: disabled_students_allowance + label: Disabled Students' Allowance + variable: disabled_students_allowance + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: bursary_fund_16_to_19 + label: 16 to 19 Bursary Fund + variable: bursary_fund_16_to_19 + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + # Public services in kind + - id: nhs_spending + label: NHS spending + variable: nhs_spending + entity: person + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + - id: dfe_education_spending + label: State education spending + variable: dfe_education_spending + entity: household + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + - id: dft_subsidy_spending + label: Transport subsidy spending + variable: dft_subsidy_spending + entity: household + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + # Cost-of-living and energy support (none in force at current law) + - id: cost_of_living_support_payment + label: Cost-of-living support payment + variable: cost_of_living_support_payment + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: energy_bills_rebate + label: Energy Bills Rebate + variable: energy_bills_rebate + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: epg_subsidy + label: Energy Price Guarantee subsidy + variable: epg_subsidy + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + # Reform-only spending + - id: basic_income + label: Basic income + variable: basic_income + entity: person + report_group: hypothetical_spending + totals: [gov_spending, household_benefits] + sign: any + zero_at_baseline: expected + - id: disability_basic_income + label: Disability basic income + variable: disability_basic_income + entity: person + report_group: hypothetical_spending + totals: [gov_spending, household_benefits] + sign: any + zero_at_baseline: expected + - id: other_public_spending_budget_change + label: Other public spending budget change + variable: other_public_spending_budget_change + entity: household + report_group: hypothetical_spending + totals: [gov_spending] + sign: any + zero_at_baseline: expected + caseload: false diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py new file mode 100644 index 0000000000..062945b2b1 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py @@ -0,0 +1,144 @@ +"""The aggregate-impact measurement on the certified dataset. + +One year, the counted-once check on every tax component plus a few benefits. +Runs where microsimulation tests run (see conftest's data mode). +""" + +import dataclasses + +import numpy as np +import pytest + +from policyengine_uk.utils.aggregate_impact.measure import measure +from policyengine_uk.utils.aggregate_impact.partition import ( + default_partition_path, + load_partition, +) +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +pytestmark = pytest.mark.microsimulation + +YEAR = 2026 +# Universal Credit deducts income tax and National Insurance on Statutory Sick +# Pay (taxable_employment_income adds employment_benefits) but does not count +# the pay itself as earnings (uc_individual_earned_income_before_mif), so UC +# rises when SSP does. When that is fixed this test fails: remove the entry. +KNOWN_RISING_WITH_INCOME = {"statutory_sick_pay"} +SPENDING_SAMPLE = { + "state_pension", + "universal_credit", + "child_benefit", + "statutory_sick_pay", +} + + +@pytest.fixture(scope="module") +def measured(): + from policyengine_uk import Microsimulation + + sim = Microsimulation() + dataset = sim.dataset + partition = load_partition(default_partition_path()) + spec = dataclasses.replace( + load_spec(), first_year=YEAR, last_year=YEAR, guard_year=YEAR + ) + tax_components = {h.variable for h in partition.tax_heads} | { + d.variable for d in partition.double_counts + } + result, households = measure( + sim, + lambda reform=None: Microsimulation(dataset=dataset, reform=reform), + partition, + spec, + label="test", + guard_variables=tax_components | SPENDING_SAMPLE, + ) + return result, households, partition, Microsimulation(dataset=dataset) + + +def checks(result, prefix): + return [c for c in result["checks"] if c["id"].startswith(prefix)] + + +def test_accounting_identities_and_closure_hold(measured): + result, _, _, _ = measured + assert result["status"] == "ok" + for prefix in ( + "closure", + "identity:", + "finite:", + "weights:", + "bounds:", + "determinism:", + ): + failing = [c for c in checks(result, prefix) if c["status"] != "pass"] + assert failing == [], failing + + +def test_every_tax_component_is_counted_exactly_once(measured): + result, _, partition, _ = measured + guard = result["guard"] + assert guard["status"] == "ran" and guard["reset_reproduces_baseline"] + rows = {r["variable"]: r for r in guard["components"]} + for head in partition.tax_heads: + row = rows[head.variable] + assert row["status"] == "pass", row + for total, k in row["coefficients"].items(): + assert k == pytest.approx(1, abs=1e-3), (head.id, total) + + +def test_declared_double_counts_are_still_double_counted_by_the_model(measured): + result, _, partition, _ = measured + rows = {r["variable"]: r for r in result["guard"]["components"]} + for declared in partition.double_counts: + present = next( + d + for d in result["guard"]["double_counts"] + if d["variable"] == declared.variable + ) + assert present["status"] == "pass", present + for total in declared.totals: + assert rows[declared.variable]["model_coefficients"][ + total + ] == pytest.approx(2, abs=1e-3) + + +def test_spending_never_rises_by_more_than_is_added_except_known_defects(measured): + result, _, _, _ = measured + rows = {r["variable"]: r for r in result["guard"]["components"]} + for variable in SPENDING_SAMPLE: + row = rows[variable] + if variable in KNOWN_RISING_WITH_INCOME: + assert ( + row["status"] == "fail" + and 1 < row["coefficients"]["gov_spending"] < 1.05 + ) + else: + assert row["status"] == "pass", row + + +def test_head_totals_agree_with_the_model_weighted_sums(measured): + result, _, partition, sim = measured + metrics = {m["id"]: m for m in result["metrics"]} + for head in partition.heads: + section = "receipts" if head.kind == "tax" else "spending" + ours = metrics[f"{section}.{head.id}"]["values"][str(YEAR)] + theirs = float(sim.calculate(head.variable, YEAR).sum()) + assert ours == pytest.approx(theirs, rel=1e-6, abs=1e3), head.id + receipts = metrics["balance.receipts"]["values"][str(YEAR)] + model = metrics["model.gov_tax"]["values"][str(YEAR)] + double_counted = sum( + metrics[f"double_counts.{d.variable}"]["values"][str(YEAR)] + for d in partition.double_counts + if "gov_tax" in d.totals + ) + assert receipts + double_counted == pytest.approx(model, rel=1e-9) + + +def test_household_detail_is_complete(measured): + _, households, _, _ = measured + columns = households[str(YEAR)] + n = len(columns["household_id"]) + assert all(len(v) == n for v in columns.values()) + assert np.all(np.isfinite(columns["income"])) + assert set(np.unique(columns["decile"])) <= set(range(-1, 11)) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py new file mode 100644 index 0000000000..d420b31662 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py @@ -0,0 +1,311 @@ +"""Comparison and rendering of two measurements, on synthetic results.""" + +import copy +import math + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact import COMMENT_MARKER, TOOL_SCHEMA +from policyengine_uk.utils.aggregate_impact.compare import ( + classify_check, + compare, + decile_impacts, +) +from policyengine_uk.utils.aggregate_impact.measure import weighted_gini +from policyengine_uk.utils.aggregate_impact.render import fmt_change, fmt_level, render +from policyengine_uk.utils.aggregate_impact.spec import Threshold, load_spec + +SPEC = load_spec() +YEARS = SPEC.years +SHA = "5" * 64 + +KINDS = { + "receipts": "gbp", + "caseloads": "count", + "poverty": "rate", + "inequality": "index", +} +finite = st.floats(-1e12, 1e12, allow_nan=False, allow_infinity=False) + + +def result(values, checks=(), sha=SHA, status="ok", partition_sha="p"): + """A measurement with one metric per (section, name) and the given checks.""" + return { + "schema": TOOL_SCHEMA, + "status": status, + "label": "x", + "years": YEARS, + "provenance": { + "dataset": {"verified_sha256": sha, "path": "d.h5"}, + "git_sha": "abc", + }, + "runtime": { + "distributions": ["numpy==2.0"], + "python": "3.13", + "machine": "arm64", + }, + "partition": {"sha256": partition_sha, "source": "s"}, + "metrics": [ + { + "id": f"{section}.{name}", + "label": name, + "section": section, + "group": None, + "kind": KINDS[section], + "unit": None, + "values": {str(y): v for y, v in zip(YEARS, series)}, + } + for (section, name), series in values.items() + ], + "checks": [ + { + "id": cid, + "label": cid, + "section": "s", + "severity": "fail", + "status": s, + "details": ["d"], + } + for cid, s in checks + ], + "guard": {"status": "skipped", "reason": "test"}, + "reforms": [], + "timings": {"total": 1}, + } + + +metric_values = st.dictionaries( + st.tuples(st.sampled_from(sorted(KINDS)), st.sampled_from(["a", "b", "c"])), + st.lists(finite, min_size=len(YEARS), max_size=len(YEARS)), + min_size=1, + max_size=6, +) + + +@pytest.mark.parametrize( + "base, head, blocking, expected", + [ + ("pass", "pass", False, "pass"), + ("fail", "pass", False, "fixed"), + ("warn", "pass", False, "fixed"), + (None, "pass", False, "pass"), + ("pass", "fail", False, "regression"), + (None, "fail", False, "regression"), + ("fail", "fail", False, "pre_existing"), + ("fail", "fail", True, "regression"), + ("warn", "fail", False, "regression"), + ("pass", "warn", False, "new_warning"), + ("warn", "warn", False, "warning"), + ("pass", None, False, "removed"), + ], +) +def test_check_classification(base, head, blocking, expected): + assert classify_check(base, head, blocking) == expected + + +@settings(max_examples=60, deadline=None) +@given(values=metric_values) +def test_identical_sides_change_nothing(values): + side = result(values, checks=[("identity:x", "pass"), ("sign:y", "fail")]) + summary = compare(side, copy.deepcopy(side), SPEC) + assert summary["status"] == "pass" + assert summary["flags"] == [] + assert all(not row["changed"] for row in summary["rows"]) + assert {c["classification"] for c in summary["checks"]} == {"pass", "pre_existing"} + + +@settings(max_examples=60, deadline=None) +@given(base=metric_values, data=st.data()) +def test_changes_are_antisymmetric(base, data): + head = {k: [v + data.draw(finite) for v in series] for k, series in base.items()} + forward = {r["id"]: r for r in compare(result(base), result(head), SPEC)["rows"]} + backward = {r["id"]: r for r in compare(result(head), result(base), SPEC)["rows"]} + for metric_id, row in forward.items(): + for year, cell in row["values"].items(): + assert cell["delta"] == pytest.approx( + -backward[metric_id]["values"][year]["delta"], abs=1e-3 + ) + + +@settings(max_examples=200, deadline=None) +@given(base=finite, head=finite, absolute=st.floats(0, 1e9), relative=st.floats(0, 0.5)) +def test_a_move_is_flagged_exactly_when_it_exceeds_the_larger_bound( + base, head, absolute, relative +): + threshold = Threshold(absolute, relative) + assert threshold.exceeded(base, head) == ( + abs(head - base) > max(absolute, relative * abs(base)) + ) + + +def test_flags_follow_the_spec_thresholds(): + base = {("receipts", "a"): [100e9] * len(YEARS)} + small = {("receipts", "a"): [100e9 + 50e6] * len(YEARS)} + large = {("receipts", "a"): [100e9 + 200e6] * len(YEARS)} + assert compare(result(base), result(small), SPEC)["flags"] == [] + assert len(compare(result(base), result(large), SPEC)["flags"]) == len(YEARS) + + +def test_a_crashed_or_missing_head_fails(): + base = result({("receipts", "a"): [1.0] * len(YEARS)}) + crashed = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "error": "Traceback\nValueError: boom", + } + summary = compare(base, crashed, SPEC) + assert summary["status"] == "fail" and "boom" in summary["failures"][0] + assert compare(base, None, SPEC)["status"] == "fail" + + +def test_without_a_base_the_heads_own_failing_checks_block(): + head = result( + {("receipts", "a"): [1.0] * len(YEARS)}, checks=[("identity:x", "fail")] + ) + summary = compare(None, head, SPEC) + assert summary["status"] == "fail" + assert any("identity:x" in f for f in summary["failures"]) + + +def test_different_datasets_fail(): + values = {("receipts", "a"): [1.0] * len(YEARS)} + summary = compare(result(values), result(values, sha="6" * 64), SPEC) + assert summary["status"] == "fail" + + +def test_non_finite_head_figures_block_even_when_the_base_shares_them(): + values = {("receipts", "a"): [1.0] * len(YEARS)} + checks = [("finite:metrics", "fail")] + summary = compare(result(values, checks), result(values, checks), SPEC) + assert summary["status"] == "fail" + + +def test_strict_mode_fails_pre_existing_failures(): + values = {("receipts", "a"): [1.0] * len(YEARS)} + side = result(values, checks=[("sign:vat", "fail")]) + assert compare(side, copy.deepcopy(side), SPEC)["status"] == "pass" + assert compare(side, copy.deepcopy(side), SPEC, strict=True)["status"] == "fail" + + +def test_a_partition_edit_is_noted(): + values = {("receipts", "a"): [1.0] * len(YEARS)} + summary = compare(result(values), result(values, partition_sha="q"), SPEC) + assert any("partition" in w for w in summary["warnings"]) + + +def households(n, rng, decile=None, income=None, weight=None): + return { + str(y): { + "household_id": np.arange(n), + "household_weight": weight + if weight is not None + else rng.uniform(1, 1000, n), + "household_count_people": rng.integers(1, 6, n).astype(float), + "decile": decile if decile is not None else rng.integers(1, 11, n), + "income": income if income is not None else rng.uniform(1e3, 1e5, n), + } + for y in YEARS + } + + +@settings(max_examples=30, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), n=st.integers(20, 400)) +def test_decile_impacts_match_a_direct_grouped_calculation(seed, n): + rng = np.random.default_rng(seed) + base = households(n, rng) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + rng.normal(0, 500, n) + impacts, _ = decile_impacts(base, head, YEARS) + year = str(YEARS[0]) + b, h = base[year], head[year] + frame = pd.DataFrame( + { + "d": b["decile"], + "w": b["household_weight"], + "base": b["income"], + "change": h["income"] - b["income"], + } + ) + for row in impacts[year][:-1]: + group = frame[frame.d == row["decile"]] + if group.empty: + continue + mean_change = (group.w * group.change).sum() / group.w.sum() + relative = (group.w * group.change).sum() / (group.w * group.base).sum() + assert row["mean_change"] == pytest.approx(mean_change) + assert row["relative_change"] == pytest.approx(relative) + + +@settings(max_examples=30, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), shift=st.floats(-1e4, 1e4)) +def test_a_uniform_change_shows_in_every_decile(seed, shift): + rng = np.random.default_rng(seed) + base = households(200, rng, decile=np.repeat(np.arange(1, 11), 20)) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + shift + columns["household_id"] = columns["household_id"][::-1].copy() + for key in ("household_weight", "household_count_people", "decile", "income"): + columns[key] = columns[key][::-1].copy() + impacts, notes = decile_impacts(base, head, YEARS) + assert not any("differ" in n for n in notes) + for row in impacts[str(YEARS[0])]: + assert row["mean_change"] == pytest.approx(shift, abs=1e-6) + assert row["share_gaining"] == (1.0 if shift > 1 else 0.0) + assert row["share_losing"] == (1.0 if shift < -1 else 0.0) + + +@settings(max_examples=50, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), n=st.integers(2, 300)) +def test_gini_matches_microdf(seed, n): + from microdf import MicroSeries + + rng = np.random.default_rng(seed) + income = rng.lognormal(10, 1, n) + weights = rng.uniform(0.5, 50, n) + assert weighted_gini(income, weights) == pytest.approx( + float(MicroSeries(income, weights=weights).gini()), abs=1e-6 + ) + + +def test_gini_bounds(): + assert weighted_gini(np.ones(10), np.ones(10)) == pytest.approx(0) + assert weighted_gini(np.r_[np.zeros(99), 1.0], np.ones(100)) == pytest.approx(0.99) + + +@pytest.mark.parametrize( + "value, kind, text", + [ + (1.234e9, "gbp", "+£1.23bn"), + (-5.5e6, "gbp", "-£5.5m"), + (0, "gbp", "0"), + (12_300, "count", "+12.3k"), + (0.0012, "rate", "+0.12pp"), + (None, "gbp", "n/a"), + ], +) +def test_change_formatting(value, kind, text): + assert fmt_change(value, kind) == text + + +def test_level_formatting(): + assert fmt_level(320.5e9, "gbp") == "£320.50bn" + assert fmt_level(0.2335, "rate") == "23.4%" + assert fmt_level(float("nan"), "gbp") == "n/a" + + +@settings(max_examples=20, deadline=None) +@given(values=metric_values, limit=st.integers(2000, 20000)) +def test_the_comment_carries_the_marker_and_fits_its_limit(values, limit): + head = {k: [v + 1e9 for v in series] for k, series in values.items()} + summary = compare(result(values), result(head), SPEC) + spec = SPEC.__class__(**{**SPEC.__dict__, "max_comment_characters": limit}) + text = render(summary, spec) + assert text.startswith(COMMENT_MARKER) + assert len(text) <= limit + 1 + assert render(summary, spec) == text diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py new file mode 100644 index 0000000000..397fcfecc2 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py @@ -0,0 +1,191 @@ +"""The certified dataset pin, the fixtures that record it, and the data mode.""" + +import hashlib +import os +from pathlib import Path + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.tests.conftest import ( + CERTIFIED_DATASET_PIN, + DATASET_ENV_VAR, + DEFAULT_TEST_DATASET_URL, + REQUIRE_DATA_ENV_VAR, + TOKEN_ENV_VAR, + plan_test_data, +) +from policyengine_uk.utils.aggregate_impact.dataset import ( + DatasetMismatchError, + DatasetPinError, + load_pin, + sha256_file, + verify_file, +) + +TESTS = Path(__file__).resolve().parents[1] +PIN = load_pin(CERTIFIED_DATASET_PIN) + + +def test_pin_is_immutable_and_complete(): + assert PIN.problems() == [] + assert len(PIN.revision) == 40 and len(PIN.sha256) == 64 + assert PIN.uri == f"hf://{PIN.repo_id}/{PIN.path}@{PIN.revision}" + assert DEFAULT_TEST_DATASET_URL == PIN.uri + + +def test_reform_fixtures_record_the_pinned_dataset(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f) + recorded = fixtures["dataset"] + assert recorded["revision"] == PIN.revision + assert recorded["sha256"] == PIN.sha256 + assert recorded["path"] == PIN.path + assert str(recorded["data_version"]) == PIN.data_version + + +def test_every_reform_fixture_keeps_its_receipts(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f) + for reform in fixtures["reforms"]: + receipts = reform["receipts"] + assert len(receipts) >= 2, reform["name"] + latest = receipts[-1] + assert PIN.data_version in latest["data"], reform["name"] + assert round(latest["impact"], 1) == reform["expected_impact"], reform["name"] + + +def _pin_file(tmp_path, **overrides): + raw = { + "repo_id": PIN.repo_id, + "path": PIN.path, + "revision": PIN.revision, + "sha256": PIN.sha256, + **overrides, + } + path = tmp_path / "pin.yaml" + path.write_text(yaml.safe_dump(raw)) + return path + + +@pytest.mark.parametrize( + "overrides", + [ + {"revision": "1.58.0"}, + {"revision": "main"}, + {"sha256": "5BA399EA"}, + {"repo_id": "no-slash"}, + {"path": "/absolute.h5"}, + {"unexpected": "key"}, + ], +) +def test_mutable_or_malformed_pins_are_rejected(tmp_path, overrides): + with pytest.raises(DatasetPinError): + load_pin(_pin_file(tmp_path, **overrides)) + + +@settings(max_examples=50, deadline=None) +@given(revision=st.text(max_size=45)) +def test_only_a_full_commit_hash_is_accepted_as_a_revision(revision): + from dataclasses import replace + + pin = replace(PIN, revision=revision) + is_commit = len(revision) == 40 and all(c in "0123456789abcdef" for c in revision) + assert (not any("revision" in p for p in pin.problems())) == is_commit + + +@settings(max_examples=30, deadline=None) +@given(content=st.binary(max_size=4096), tamper=st.integers(0, 4095)) +def test_verification_accepts_the_pinned_bytes_and_rejects_any_other( + tmp_path_factory, content, tamper +): + path = tmp_path_factory.mktemp("data") / "file.h5" + path.write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + assert sha256_file(path) == digest + assert verify_file(path, digest) == digest + if content: + changed = bytearray(content) + changed[tamper % len(content)] ^= 0xFF + path.write_bytes(bytes(changed)) + with pytest.raises(DatasetMismatchError): + verify_file(path, digest) + + +ENV_VALUES = st.one_of( + st.none(), st.sampled_from(["", "0", "1", "true", "FALSE", "yes", "maybe"]) +) +DATASETS = st.one_of( + st.none(), st.just(""), st.just(PIN.uri), st.just("hf://other/repo/file.h5@abc") +) +TOKENS = st.one_of(st.none(), st.just(""), st.just("hf_token")) + + +def _env(flag, dataset, token, extra): + env = dict(extra) + for key, value in ( + (REQUIRE_DATA_ENV_VAR, flag), + (DATASET_ENV_VAR, dataset), + (TOKEN_ENV_VAR, token), + ): + if value is not None: + env[key] = value + return env + + +@settings(max_examples=300, deadline=None) +@given( + flag=ENV_VALUES, + dataset=DATASETS, + token=TOKENS, + extra=st.dictionaries(st.sampled_from(["HOME", "CI", "PATH"]), st.text(max_size=5)), +) +def test_data_plan_invariants(flag, dataset, token, extra): + plan = plan_test_data(_env(flag, dataset, token, extra), PIN.uri) + assert plan.action in ("run", "skip", "fail") + # Mandatory mode never skips, and runs only on the pinned dataset. + if plan.mandatory: + assert plan.action != "skip" + if plan.action == "run": + assert plan.pinned and plan.dataset == PIN.uri + # Running always names a dataset; an explicit one always wins. + if plan.action == "run": + assert plan.dataset + if dataset: + assert plan.dataset == dataset + # Nothing configured and not mandatory: the old behaviour, a skip. + if ( + not dataset + and not token + and str(flag or "").strip().lower() in ("", "0", "false", "no") + ): + assert plan.action == "skip" + # An unrecognised mode value fails closed. + if flag is not None and flag.strip().lower() not in ( + "", + "0", + "1", + "true", + "false", + "yes", + "no", + "on", + "off", + ): + assert plan.action == "fail" + # Unrelated variables never change the plan. + assert plan == plan_test_data(_env(flag, dataset, token, {}), PIN.uri) + + +def test_data_plan_examples(): + assert plan_test_data({TOKEN_ENV_VAR: "t"}, PIN.uri).action == "run" + assert plan_test_data({}, PIN.uri).action == "skip" + assert plan_test_data({REQUIRE_DATA_ENV_VAR: "1"}, PIN.uri).action == "fail" + assert plan_test_data( + {REQUIRE_DATA_ENV_VAR: "1", TOKEN_ENV_VAR: "t"}, PIN.uri + ).pinned + other = {REQUIRE_DATA_ENV_VAR: "1", DATASET_ENV_VAR: "local.h5"} + assert plan_test_data(other, PIN.uri).action == "fail" + assert plan_test_data({DATASET_ENV_VAR: "local.h5"}, PIN.uri).action == "run" diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py new file mode 100644 index 0000000000..6f98e66ad6 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py @@ -0,0 +1,212 @@ +"""The counted-once check, on a small fake simulation whose formulas are known. + +The fake implements only the simulation API the check uses, so each case +controls exactly which component contains or reads which. +""" + +from types import SimpleNamespace + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact.guard import counted_once, snapshot_inputs +from policyengine_uk.utils.aggregate_impact.partition import parse_partition + +YEAR = 2026 + + +class FakeHolder: + def __init__(self, sim, variable): + self.sim, self.variable = sim, variable + + def get_known_periods(self): + return [p for (v, p) in self.sim.inputs if v == self.variable] + + def get_array(self, period): + return self.sim.inputs[(self.variable, str(period))] + + +class FakeSim: + """Household-level variables only; formulas read each other by name.""" + + def __init__(self, formulas, inputs, weights): + self.formulas = formulas + self.inputs = { + (k, str(YEAR)): np.asarray(v, dtype=float) for k, v in inputs.items() + } + self.inputs[("household_weight", str(YEAR))] = np.asarray(weights, dtype=float) + self.cache = {} + names = set(formulas) | set(inputs) | {"household_weight"} + self.tax_benefit_system = SimpleNamespace( + variables={ + name: SimpleNamespace( + entity=SimpleNamespace(key="household"), + definition_period="year", + value_type=float, + ) + for name in names + } + ) + + def calculate(self, variable, period, unweighted=True, map_to=None): + key = (variable, str(period)) + if key in self.inputs: + return self.inputs[key] + if key not in self.cache: + self.cache[key] = np.asarray( + self.formulas[variable](lambda name: self.calculate(name, period)), + dtype=float, + ) + return self.cache[key] + + def map_result(self, values, source, target): + assert source == target == "household" + return values + + def set_input(self, variable, period, values): + self.inputs[(variable, str(period))] = np.asarray(values, dtype=float) + + def delete_arrays(self, variable, period=None): + for store in (self.inputs, self.cache): + for key in [k for k in store if k[0] == variable]: + del store[key] + + def get_holder(self, variable): + return FakeHolder(self, variable) + + def _invalidate_all_caches(self): + self.cache = {} + + +def partition(tax_heads, spending_heads=(), double_counts=()): + def heads(spec, total): + return [ + { + "id": v, + "label": v, + "variable": v, + "totals": [total], + "entity": "household", + } + for v in spec + ] + + return parse_partition( + { + "schema_version": 1, + "totals": {"gov_tax": {}, "gov_spending": {}}, + "tax_heads": heads(tax_heads, "gov_tax"), + "spending_heads": heads(spending_heads, "gov_spending"), + "model_double_counts": [ + {"variable": v, "contained_in": c, "totals": ["gov_tax"]} + for v, c in double_counts + ], + } + ) + + +def model(contains=True, interaction=-0.5): + """tax_a may contain tax_b; ben_d reads ben_c with the given slope.""" + return { + "tax_b": lambda f: 5.0 + 0 * f("x"), + "tax_a": lambda f: 0.1 * f("x") + (f("tax_b") if contains else 0), + "gov_tax": lambda f: f("tax_a") + f("tax_b"), + "ben_c": lambda f: 100 + 0 * f("x"), + "ben_d": lambda f: np.maximum(0, 200 + interaction * f("ben_c")), + "gov_spending": lambda f: f("ben_c") + f("ben_d"), + } + + +def run(formulas, part, x=(1000, 2000, 3000), weights=(1.5, 2.0, 0.5)): + sim = FakeSim(formulas, {"x": x}, weights) + snapshot = snapshot_inputs(sim, part.component_variables()) + result = counted_once(sim, part, YEAR, 1e-6, snapshot) + return sim, result + + +def by_variable(result): + return {r["variable"]: r for r in result["components"]} + + +def test_declared_double_count_passes_and_the_model_total_counts_it_twice(): + _, result = run( + model(contains=True), + partition(["tax_a"], ["ben_c", "ben_d"], [("tax_b", "tax_a")]), + ) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "pass" + assert rows["tax_b"]["coefficients"]["gov_tax"] == pytest.approx(1) + assert rows["tax_b"]["model_coefficients"]["gov_tax"] == pytest.approx(2) + assert result["double_counts"][0]["status"] == "pass" + assert result["reset_reproduces_baseline"] + + +def test_stale_double_count_declaration_fails(): + _, result = run( + model(contains=False), partition(["tax_a"], [], [("tax_b", "tax_a")]) + ) + assert result["double_counts"][0]["status"] == "fail" + assert result["double_counts"][0]["coefficient"] == pytest.approx(0) + + +def test_undeclared_containment_fails_with_coefficient_two(): + _, result = run(model(contains=True), partition(["tax_a", "tax_b"])) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "fail" + assert rows["tax_b"]["coefficients"]["gov_tax"] == pytest.approx(2) + assert rows["tax_a"]["status"] == "pass" + + +def test_means_tested_offset_passes_but_a_benefit_rising_with_income_fails(): + _, offset = run(model(interaction=-0.5), partition(["tax_a"], ["ben_c", "ben_d"])) + assert by_variable(offset)["ben_c"]["coefficients"][ + "gov_spending" + ] == pytest.approx(0.5) + assert by_variable(offset)["ben_c"]["status"] == "pass" + _, rising = run(model(interaction=0.2), partition(["tax_a"], ["ben_c", "ben_d"])) + assert by_variable(rising)["ben_c"]["coefficients"][ + "gov_spending" + ] == pytest.approx(1.2) + assert by_variable(rising)["ben_c"]["status"] == "fail" + + +def test_supplied_inputs_are_restored_after_perturbation(): + formulas = model(contains=False) + formulas.pop("tax_b") + sim = FakeSim(formulas, {"x": (1, 2, 3), "tax_b": (7, 0, 9)}, (1, 1, 1)) + part = partition(["tax_a", "tax_b"]) + snapshot = snapshot_inputs(sim, part.component_variables()) + result = counted_once(sim, part, YEAR, 1e-6, snapshot) + assert np.array_equal(sim.inputs[("tax_b", str(YEAR))], [7, 0, 9]) + assert result["reset_reproduces_baseline"] + assert by_variable(result)["tax_b"]["units"] == 2 + + +def test_zero_spending_component_is_skipped_but_zero_tax_is_tested(): + formulas = model(contains=False) + formulas["tax_b"] = lambda f: 0 * f("x") + formulas["ben_c"] = lambda f: 0 * f("x") + _, result = run(formulas, partition(["tax_a", "tax_b"], ["ben_c", "ben_d"])) + rows = by_variable(result) + assert rows["ben_c"]["status"] == "skipped" + assert rows["tax_b"]["status"] == "pass" and rows["tax_b"]["units"] == 3 + + +@settings(max_examples=40, deadline=None) +@given( + x=st.lists(st.floats(1, 1e6), min_size=1, max_size=8), + weight=st.floats(0.1, 1e4), + contains=st.booleans(), +) +def test_coefficient_is_two_exactly_when_a_head_contains_another(x, weight, contains): + _, result = run( + model(contains=contains), + partition(["tax_a", "tax_b"]), + x=x, + weights=[weight] * len(x), + ) + k = by_variable(result)["tax_b"]["coefficients"]["gov_tax"] + assert k == pytest.approx(2 if contains else 1, rel=1e-6) + assert result["reset_reproduces_baseline"] diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py new file mode 100644 index 0000000000..ab932f37d3 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py @@ -0,0 +1,203 @@ +"""The fiscal partition against the model's own component lists. + +These run without data. The partition was written from the receipts +classification and each variable's formula; closure compares it with +GOV_TAX_VARIABLES and its siblings, which are a second, independent statement +of the same totals. +""" + +import copy + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem +from policyengine_uk.utils.aggregate_impact.partition import ( + PartitionError, + closure_problems, + default_partition_path, + group_label, + load_partition, + parse_partition, + read_model_lists, + structural_problems, +) + +PATH = default_partition_path() + + +@pytest.fixture(scope="module") +def system(): + return CountryTaxBenefitSystem() + + +@pytest.fixture(scope="module") +def shipped(): + return load_partition(PATH) + + +@pytest.fixture(scope="module") +def raw(): + with open(PATH) as f: + return yaml.safe_load(f) + + +@pytest.fixture(scope="module") +def model_lists(shipped): + lists, problems = read_model_lists(shipped) + assert not problems + return lists + + +def test_shipped_partition_is_found_and_loads(shipped): + assert PATH is not None and PATH.name == "fiscal_partition.yaml" + assert shipped.schema_version == 1 + assert shipped.tax_heads and shipped.spending_heads + + +def test_every_model_total_is_exactly_its_heads_and_declared_double_counts( + shipped, model_lists, system +): + assert set(model_lists) == { + "gov_tax", + "household_tax", + "pre_budget_change_household_tax", + "gov_spending", + "household_benefits", + } + assert closure_problems(shipped, model_lists, system.variables) == [] + + +def test_every_head_is_a_money_variable_with_its_declared_entity(shipped, system): + for head in shipped.heads: + variable = system.variables[head.variable] + assert variable.value_type is float, head.id + assert variable.definition_period in ("year", "month"), head.id + assert head.entity == variable.entity.key, head.id + + +def test_receipts_and_spending_are_separate(shipped): + for head in shipped.tax_heads: + assert not set(head.totals) & {"gov_spending", "household_benefits"} + for head in shipped.spending_heads: + assert not set(head.totals) & {"gov_tax", "household_tax"} + + +def test_dropped_when_matches_the_council_tax_abolition_formulas(shipped): + switch = "gov.contrib.abolish_council_tax" + assert shipped.dropped("gov_tax", [switch]) == { + "council_tax", + "high_value_council_tax_surcharge", + } + assert shipped.dropped("gov_spending", [switch]) == {"council_tax_benefit"} + assert shipped.dropped("gov_tax") == set() + assert set(shipped.components("gov_tax")) - set( + shipped.components("gov_tax", [switch]) + ) == {"council_tax", "high_value_council_tax_surcharge"} + + +def test_group_labels_read_as_sentence_case(): + assert group_label("excise_duties") == "Excise duties" + assert group_label("Income tax and NICs") == "Income tax and NICs" + + +def _remove_head(raw, head_id): + """The partition without one head and every reference to it.""" + out = copy.deepcopy(raw) + removed = next( + h for h in raw["tax_heads"] + raw["spending_heads"] if h["id"] == head_id + ) + for key in ("tax_heads", "spending_heads"): + out[key] = [h for h in out[key] if h["id"] != head_id] + for head in out[key]: + if head.get("reads"): + head["reads"] = [r for r in head["reads"] if r.get("head") != head_id] + out["exclusions"] = [ + e + for e in out.get("exclusions") or [] + if e.get("head") != head_id and head_id not in (e.get("spans") or []) + ] + out["model_double_counts"] = [ + d + for d in out.get("model_double_counts") or [] + if d["contained_in"] != removed["variable"] + ] + for spec in (out.get("totals") or {}).values(): + for parameter, variables in ((spec or {}).get("dropped_when") or {}).items(): + spec["dropped_when"][parameter] = [ + v for v in variables if v != removed["variable"] + ] + return out + + +def _mutations(raw): + """Each yields a partition dict that a check must reject, and which check.""" + out = [] + a = copy.deepcopy(raw) + a["tax_heads"][1]["variable"] = a["tax_heads"][0]["variable"] + out.append(("duplicate head variable", a, "structural")) + out.append( + ("missing head", _remove_head(raw, raw["tax_heads"][0]["id"]), "closure") + ) + c = copy.deepcopy(raw) + c["spending_heads"].append( + { + "id": "not_a_model_component", + "label": "x", + "variable": "employment_income", + "totals": ["gov_spending"], + } + ) + out.append(("extra component", c, "closure")) + d = copy.deepcopy(raw) + if d.get("model_double_counts"): + d["model_double_counts"][0]["contained_in"] = "vat" + out.append(("double count in a total its container is not in", d, "structural")) + e = copy.deepcopy(raw) + e["tax_heads"][0]["totals"] = list(e["tax_heads"][0]["totals"]) + ["gov_spending"] + out.append(("tax head in a spending total", e, "structural")) + f = copy.deepcopy(raw) + f["tax_heads"][0]["id"] = f["tax_heads"][1]["id"] + out.append(("duplicate head id", f, "structural")) + return out + + +def test_each_kind_of_broken_partition_is_rejected(raw, model_lists, system): + for name, mutated, check in _mutations(raw): + if check == "structural": + with pytest.raises(PartitionError): + parse_partition(mutated) + else: + partition = parse_partition(mutated) + assert closure_problems(partition, model_lists, system.variables), name + + +@settings(max_examples=30, deadline=None) +@given(data=st.data()) +def test_component_sets_do_not_depend_on_head_order(raw, data): + shuffled = copy.deepcopy(raw) + shuffled["tax_heads"] = data.draw(st.permutations(shuffled["tax_heads"])) + shuffled["spending_heads"] = data.draw(st.permutations(shuffled["spending_heads"])) + a, b = parse_partition(raw), parse_partition(shuffled) + for total in a.totals: + assert sorted(a.components(total)) == sorted(b.components(total)) + + +@settings(max_examples=30, deadline=None) +@given(data=st.data()) +def test_removing_any_head_breaks_closure_of_each_total_it_was_in( + raw, model_lists, data +): + heads = raw["tax_heads"] + raw["spending_heads"] + removed = heads[data.draw(st.integers(0, len(heads) - 1))] + problems = closure_problems( + parse_partition(_remove_head(raw, removed["id"])), model_lists + ) + for total in removed["totals"]: + assert any(total in p and removed["variable"] in p for p in problems) + + +def test_structural_checks_pass_on_the_shipped_file(shipped): + assert structural_problems(shipped) == [] diff --git a/policyengine_uk/tests/certified_dataset.yaml b/policyengine_uk/tests/certified_dataset.yaml new file mode 100644 index 0000000000..994f08b5ba --- /dev/null +++ b/policyengine_uk/tests/certified_dataset.yaml @@ -0,0 +1,20 @@ +# The dataset that credentialed test jobs and the aggregate-impact report run +# on. It is pinned by Hugging Face commit, not by tag, and every run checks the +# file's SHA-256 against `sha256`, so the selection cannot move underneath a +# test. Change it only to another certified pair, together with the reform +# regression fixtures in microsimulation/reforms_config.yaml, which record the +# same dataset. +data_package: policyengine-uk-data +data_version: 1.58.0 +repo_id: policyengine/policyengine-uk-data-private +path: enhanced_frs_2024_25.h5 +# Commit the 1.58.0 tag pointed to on 2026-10-11. +revision: 47bbb58d3f6467aa4a7059bc6110d2bdcf699b29 +hf_tag: 1.58.0 +sha256: 5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6 +size_bytes: 128369110 +certification: + certified_by: policyengine.py bundle uk-6.2.6 (PolicyEngine/policyengine.py#555) + certified_for_model_version: 2.125.1 + built_with_model_version: 2.122.2 + release_manifest_revision: fdeba38517bd86067f34c0a5d8ed8923efb3b10a diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index d611a0c1c5..d116d5f814 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -2,18 +2,144 @@ import os import re import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Optional import pytest from policyengine_core.taxbenefitsystems import TaxBenefitSystem -DEFAULT_TEST_DATASET_URL = ( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.3" -) +from policyengine_uk.utils.aggregate_impact.dataset import load_pin, materialise -if os.environ.get("HUGGING_FACE_TOKEN") and not os.environ.get( - "POLICYENGINE_UK_DEFAULT_DATASET" -): - os.environ["POLICYENGINE_UK_DEFAULT_DATASET"] = DEFAULT_TEST_DATASET_URL +# Microsimulation tests run on one certified dataset, pinned by commit and +# SHA-256 in certified_dataset.yaml. +CERTIFIED_DATASET_PIN = Path(__file__).with_name("certified_dataset.yaml") +CERTIFIED_DATASET = load_pin(CERTIFIED_DATASET_PIN) +DEFAULT_TEST_DATASET_URL = CERTIFIED_DATASET.uri + +DATASET_ENV_VAR = "POLICYENGINE_UK_DEFAULT_DATASET" +TOKEN_ENV_VAR = "HUGGING_FACE_TOKEN" +# Mandatory-data mode, for CI jobs that must run the microsimulation tests: +# with it set, a test that needs data fails when the certified dataset is +# unavailable, instead of being skipped, and runs only on that dataset. +REQUIRE_DATA_ENV_VAR = "POLICYENGINE_UK_REQUIRE_TEST_DATA" +_TRUE = {"1", "true", "yes", "on"} +_FALSE = {"", "0", "false", "no", "off"} + + +@dataclass(frozen=True) +class TestDataPlan: + """What the microsimulation tests run on, decided from the environment.""" + + __test__ = False # not a test class + + action: str # "run", "skip" or "fail" + dataset: Optional[str] + pinned: bool + mandatory: bool + reason: str + + +def plan_test_data(env: Mapping[str, str], pinned_uri: str) -> TestDataPlan: + """Decide how microsimulation tests get their data. + + - An explicit ``POLICYENGINE_UK_DEFAULT_DATASET`` wins; in mandatory mode + it must be the pinned certified dataset. + - Otherwise a Hugging Face token selects the pinned certified dataset. + - Otherwise the tests skip, or in mandatory mode fail. + + An unrecognised mandatory-mode value fails closed. + """ + flag = str(env.get(REQUIRE_DATA_ENV_VAR, "")).strip().lower() + explicit = env.get(DATASET_ENV_VAR) or None + token = env.get(TOKEN_ENV_VAR) or None + if flag not in _TRUE | _FALSE: + return TestDataPlan( + "fail", + explicit, + False, + True, + f"{REQUIRE_DATA_ENV_VAR}={env.get(REQUIRE_DATA_ENV_VAR)!r} is not a " + "recognised true or false value", + ) + mandatory = flag in _TRUE + if explicit: + pinned = explicit == pinned_uri + if mandatory and not pinned: + return TestDataPlan( + "fail", + explicit, + False, + True, + f"mandatory-data mode runs only on the pinned certified dataset " + f"{pinned_uri}, not {explicit}", + ) + return TestDataPlan( + "run", + explicit, + pinned, + mandatory, + "pinned certified dataset" if pinned else "dataset set explicitly", + ) + if token: + return TestDataPlan( + "run", pinned_uri, True, mandatory, "pinned certified dataset" + ) + if mandatory: + return TestDataPlan( + "fail", + None, + False, + True, + f"mandatory-data mode needs {TOKEN_ENV_VAR} (or {DATASET_ENV_VAR} set " + "to the pinned certified dataset)", + ) + return TestDataPlan( + "skip", + None, + False, + False, + f"Requires {DATASET_ENV_VAR} or {TOKEN_ENV_VAR} for microsimulation " + "dataset access", + ) + + +TEST_DATA_PLAN = plan_test_data(os.environ, DEFAULT_TEST_DATASET_URL) + +if TEST_DATA_PLAN.action == "run" and not os.environ.get(DATASET_ENV_VAR): + os.environ[DATASET_ENV_VAR] = TEST_DATA_PLAN.dataset + + +_verification: dict = {} + + +def verify_certified_dataset() -> Optional[str]: + """Fetch the pinned dataset once per process and check its SHA-256. + + Returns None when the file matches, or why it could not be verified. + """ + if "problem" not in _verification: + try: + materialise(CERTIFIED_DATASET) + _verification["problem"] = None + except Exception as error: # noqa: BLE001 - every cause fails the tests + _verification["problem"] = ( + f"The pinned certified dataset {CERTIFIED_DATASET.describe()} could " + f"not be fetched and verified: {type(error).__name__}: {error}" + ) + return _verification["problem"] + + +def needs_data(item) -> bool: + # The keyword also matches every test in the microsimulation directory. + return "microsimulation" in item.keywords + + +def pytest_report_header(config): + plan = TEST_DATA_PLAN + mode = "mandatory" if plan.mandatory else "optional" + target = CERTIFIED_DATASET.describe() if plan.pinned else (plan.dataset or "none") + return f"microsimulation data ({mode}): {plan.action} on {target} ({plan.reason})" @pytest.fixture(scope="module") @@ -30,19 +156,34 @@ def cloned_uk_tax_benefit_system(): def pytest_collection_modifyitems(config, items): - has_default_dataset = bool(os.environ.get("POLICYENGINE_UK_DEFAULT_DATASET")) - if has_default_dataset: + """Without data, microsimulation tests skip, unless data is mandatory.""" + if TEST_DATA_PLAN.action != "skip": return + skip = pytest.mark.skip(reason=TEST_DATA_PLAN.reason) + for item in items: + if needs_data(item): + item.add_marker(skip) + + +def pytest_runtest_setup(item): + """Fail data tests in mandatory mode without data; verify pinned bytes. - skip_microsimulation = pytest.mark.skip( - reason=( - "Requires POLICYENGINE_UK_DEFAULT_DATASET or HUGGING_FACE_TOKEN " - "for microsimulation dataset access" + A failure here is reported as an error in the test's setup, so the run + fails rather than passing with the test unexecuted. + """ + if not needs_data(item): + return + plan = TEST_DATA_PLAN + if plan.action == "fail": + pytest.fail( + f"Mandatory-data mode: {plan.reason}. This test needs the certified " + "dataset and must not be skipped.", + pytrace=False, ) - ) - for item in items: - if "microsimulation" in item.keywords: - item.add_marker(skip_microsimulation) + if plan.action == "run" and plan.pinned: + problem = verify_certified_dataset() + if problem: + pytest.fail(problem, pytrace=False) # policyengine-core loads every variable file of each new tax-benefit system as diff --git a/policyengine_uk/tests/microsimulation/reforms_config.yaml b/policyengine_uk/tests/microsimulation/reforms_config.yaml index 57bcc7d2ef..c48a59ac45 100644 --- a/policyengine_uk/tests/microsimulation/reforms_config.yaml +++ b/policyengine_uk/tests/microsimulation/reforms_config.yaml @@ -2,38 +2,135 @@ # These expected impacts are derived from current model output and are used to # catch unintended behavior changes. They are not external fiscal estimates # unless a reform entry is explicitly annotated otherwise. +# +# Each impact is the change in weighted gov_balance in 2029, in £bn, on the +# dataset below, which must be the one tests/certified_dataset.yaml pins. +# `receipts` keeps every earlier value with the model and data that produced +# it, so a change of dataset and a change of model stay separable. Tests ran +# on whatever enhanced FRS was latest until 2026-04-05, then on uk-data 1.40.3 +# (enhanced_frs_2023_24), and from 2026-10-11 on the certified 1.58.0 pair. The +# 2026-10-11 receipts on 1.40.3 show how much of each move is the data rather +# than the model. +dataset: + data_version: 1.58.0 + path: enhanced_frs_2024_25.h5 + revision: 47bbb58d3f6467aa4a7059bc6110d2bdcf699b29 + sha256: 5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6 reforms: - name: Raise basic rate by 1pp - expected_impact: 7.8 + expected_impact: 7.1 parameters: gov.hmrc.income_tax.rates.uk[0].rate: 0.21 + receipts: + - impact: 7.8 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.34.5 (a1832dc77, #1205, 2025-07-11) + - impact: 7.346 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 7.132 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise higher rate by 1pp - expected_impact: 4.8 + expected_impact: 4.6 parameters: gov.hmrc.income_tax.rates.uk[1].rate: 0.42 + receipts: + - impact: 4.8 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.38.1 (fa6eb613e, #1231, 2025-07-15) + - impact: 3.961 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 4.609 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise personal allowance by ~800GBP/year - expected_impact: -4.2 + expected_impact: -4.1 parameters: gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 + receipts: + - impact: -4.2 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.59.0 (53c38d229, #1396, 2025-11-27) + - impact: -4.305 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -4.135 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise child benefit by 25GBP/week per additional child - expected_impact: -1.2 + expected_impact: -1.4 parameters: gov.hmrc.child_benefit.amount.additional: 25 + receipts: + - impact: -1.2 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.38.0 (bda6d2ddf, #1228, 2025-07-14) + - impact: -1.222 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -1.428 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Reduce Universal Credit taper rate to 20% - expected_impact: -18.1 + expected_impact: -21.3 tolerance: 3.0 parameters: gov.dwp.universal_credit.means_test.reduction_rate: 0.2 + receipts: + - impact: -18.1 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.78.0 (e4be118a7, #1572, 2026-04-11) + - impact: -18.354 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -21.266 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise Class 1 main employee NICs rate to 10% - expected_impact: 13.0 + expected_impact: 12.6 parameters: gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 + receipts: + - impact: 13.0 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.59.0 (53c38d229, #1396, 2025-11-27) + - impact: 13.263 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 12.566 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise VAT standard rate by 2pp - expected_impact: 21.3 + expected_impact: 34.5 tolerance: 1.5 parameters: gov.hmrc.vat.standard_rate: 0.22 + # The move from 21.3 is the data: on 1.58.0 recorded full-rate consumption + # is 62% higher than on 1.40.3, and the VAT coverage factor (#1996, #2187) + # was set on older data. + receipts: + - impact: 21.3 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.75.3 (bc59ba950, #1540, 2026-03-29) + - impact: 21.222 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 34.478 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise additional rate by 3pp - expected_impact: 4.4 + expected_impact: 3.8 parameters: gov.hmrc.income_tax.rates.uk[2].rate: 0.48 + receipts: + - impact: 4.4 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.35.0 (88bdf190b, #1207, 2025-07-11) + - impact: 3.993 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 3.756 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) diff --git a/policyengine_uk/utils/aggregate_impact/__init__.py b/policyengine_uk/utils/aggregate_impact/__init__.py new file mode 100644 index 0000000000..e8c17a26e6 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/__init__.py @@ -0,0 +1,14 @@ +"""Base-versus-head aggregate impact report on a certified dataset. + +The report measures one model checkout at a time (``measure``), on a dataset +whose bytes are verified against a pinned SHA-256, and then compares two +measurements (``compare``). A pull request's CI measures its base and its head +on the same dataset and posts the comparison as a comment. + +This package must only import ``policyengine_uk`` through its public +simulation API, and must import its own modules relatively: CI runs the pull +request head's copy of it against the base checkout's model. +""" + +TOOL_SCHEMA = "policyengine-uk-aggregate-impact/1" +COMMENT_MARKER = "" diff --git a/policyengine_uk/utils/aggregate_impact/__main__.py b/policyengine_uk/utils/aggregate_impact/__main__.py new file mode 100644 index 0000000000..999c9f006e --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/__main__.py @@ -0,0 +1,196 @@ +"""Command line: ``measure`` one checkout, then ``compare`` two measurements. + + python -m policyengine_uk.utils.aggregate_impact measure --label head \\ + --pin policyengine_uk/tests/certified_dataset.yaml \\ + --reforms policyengine_uk/tests/microsimulation/reforms_config.yaml \\ + --out out/head + python -m policyengine_uk.utils.aggregate_impact compare \\ + --base out/base --head out/head --markdown comment.md --summary summary.json + +``measure`` exits 1 if the measurement crashed, after writing a result that +records the crash; ``compare`` exits 1 when the change fails the report. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import traceback +from pathlib import Path + +import yaml + +from . import TOOL_SCHEMA + + +def _sha256(path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def cmd_measure(args) -> int: + from .dataset import load_pin, materialise, verify_file + from .measure import write_result + + provenance = { + "git_sha": args.git_sha, + "git_ref": args.git_ref, + "tool": str(Path(__file__).resolve().parent), + } + try: + from .measure import measure, tool_digest + from .partition import default_partition_path, load_partition + from .spec import DEFAULT_SPEC_PATH, load_spec + + provenance["tool_sha256"] = tool_digest() + provenance["metrics_version"] = ( + f"{TOOL_SCHEMA}+{provenance['tool_sha256'][:16]}" + ) + + spec_path = Path(args.spec) if args.spec else DEFAULT_SPEC_PATH + spec = load_spec(spec_path, years=args.years) + provenance["spec_sha256"] = _sha256(spec_path) + + pin = load_pin(args.pin) + if args.dataset_file: + dataset_path = Path(args.dataset_file) + verified = verify_file(dataset_path, pin.sha256) + else: + dataset_path = materialise(pin) + verified = pin.sha256 + provenance["dataset"] = {**pin.as_dict(), "verified_sha256": verified} + + if args.partition: + partition_path, partition_source = Path(args.partition), "given" + else: + partition_path = default_partition_path() + partition_source = "measured model" + if partition_path is None: + if not args.fallback_partition: + raise FileNotFoundError( + "the measured policyengine_uk ships no fiscal partition; " + "pass --fallback-partition" + ) + partition_path, partition_source = ( + Path(args.fallback_partition), + "fallback", + ) + partition = load_partition(partition_path) + provenance["partition_source"] = partition_source + + reforms = [] + if args.reforms: + with open(args.reforms) as f: + reforms = (yaml.safe_load(f) or {}).get("reforms", []) + provenance["reforms_sha256"] = _sha256(args.reforms) + + from policyengine_uk import Microsimulation + + sim = Microsimulation(dataset=str(dataset_path)) + dataset_object = sim.dataset + + def factory(reform=None): + return Microsimulation(dataset=dataset_object, reform=reform or None) + + result, households = measure( + sim, + factory, + partition, + spec, + label=args.label, + reforms=reforms, + run_guard=not args.no_guard, + provenance=provenance, + ) + write_result(args.out, result, households) + print( + f"{args.label}: measured {len(result['metrics'])} figures in {result['timings']['total']}s" + ) + return 0 + except Exception: # noqa: BLE001 - recorded for the comparison, then reported + crash = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "label": args.label, + "provenance": provenance, + "error": traceback.format_exc(), + } + write_result(args.out, crash) + print(crash["error"], file=sys.stderr) + return 1 + + +def cmd_compare(args) -> int: + from .compare import compare + from .measure import read_result + from .render import render + from .spec import DEFAULT_SPEC_PATH, load_spec + + spec = load_spec(Path(args.spec) if args.spec else DEFAULT_SPEC_PATH) + base, base_households = read_result(args.base) if args.base else (None, None) + head, head_households = read_result(args.head) + summary = compare( + base, + head, + spec, + base_households=base_households, + head_households=head_households, + strict=args.strict, + ) + markdown = render(summary, spec, context={"run_url": args.run_url}) + if args.markdown: + Path(args.markdown).write_text(markdown) + if args.summary: + with open(args.summary, "w") as f: + json.dump(summary, f, indent=1, allow_nan=True, default=str) + print(f"aggregate impact: {summary['status']}") + for failure in summary["failures"]: + print(f"FAIL: {failure}") + return 1 if summary["status"] == "fail" else 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk.utils.aggregate_impact" + ) + sub = parser.add_subparsers(dest="command", required=True) + + m = sub.add_parser("measure", help="measure one model checkout") + m.add_argument("--label", required=True) + m.add_argument("--pin", required=True, help="dataset pin YAML") + m.add_argument("--dataset-file", help="local copy of the pinned file (verified)") + m.add_argument("--partition", help="fiscal partition YAML to use") + m.add_argument( + "--fallback-partition", + help="partition to use if the measured model ships none", + ) + m.add_argument("--spec", help="report spec YAML") + m.add_argument("--years", help="override the horizon, e.g. 2025-2031") + m.add_argument("--reforms", help="reform regression YAML (a 'reforms' list)") + m.add_argument("--git-sha", default="") + m.add_argument("--git-ref", default="") + m.add_argument( + "--no-guard", action="store_true", help="skip the counted-once check" + ) + m.add_argument("--out", required=True) + m.set_defaults(func=cmd_measure) + + c = sub.add_parser("compare", help="compare a base and a head measurement") + c.add_argument("--base") + c.add_argument("--head", required=True) + c.add_argument("--spec") + c.add_argument("--markdown") + c.add_argument("--summary") + c.add_argument("--run-url", default="") + c.add_argument( + "--strict", action="store_true", help="fail on checks failing on both sides" + ) + c.set_defaults(func=cmd_compare) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk/utils/aggregate_impact/api.py b/policyengine_uk/utils/aggregate_impact/api.py new file mode 100644 index 0000000000..dc3058c14f --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/api.py @@ -0,0 +1,145 @@ +"""In-process entry point for qualification (policyengine.py, workstream W4). + +``compute_budget_metrics`` measures one simulation and returns a long-format +report: one row per metric and year, plus the identity and partition checks. +Two pairs are comparable when their reports share ``metrics_version`` (the +same tool code and spec), whatever model each pair runs; each model's own +fiscal partition is read from the measured package, and its digest is +``partition_version``. + +To hold the definitions fixed across models, copy this package directory +from one policyengine-uk commit and import the copy under another name: it +imports only its own modules and policyengine_uk's public simulation API. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Callable, Optional, Sequence + +from . import TOOL_SCHEMA +from .measure import measure, tool_digest +from .partition import Partition, default_partition_path, load_partition +from .spec import DEFAULT_SPEC_PATH, Spec, load_spec + +KIND_BY_SECTION = { + "receipts": "receipt", + "spending": "spending", + "caseloads": "caseload", + "poverty": "poverty", + "poverty_count": "poverty_count", + "inequality": "inequality", + "deciles": "decile", + "balance": "balance", + "model_totals": "model_total", + "double_counts": "double_count", + "household_totals": "household_total", + "population": "population", + "reforms": "reform", +} +PARTITIONED_SECTIONS = {"receipts", "spending", "caseloads"} + + +def _spec_for_years(spec: Spec, years: Optional[Sequence[int]]) -> Spec: + if not years: + return spec + years = sorted(int(y) for y in years) + guard_year = spec.guard_year if spec.guard_year in years else years[0] + return dataclasses.replace( + spec, first_year=years[0], last_year=years[-1], guard_year=guard_year + ) + + +def to_report(result: dict) -> dict: + """Flatten a measurement into the long-format qualification report.""" + provenance = result.get("provenance") or {} + report = { + "schema": TOOL_SCHEMA + "/report", + "status": result.get("status"), + "metrics_version": provenance.get("metrics_version"), + "partition_version": (result.get("partition") or {}).get("sha256"), + "years": result.get("years"), + "metrics": [], + "identities": [], + } + if result.get("status") != "ok": + report["error"] = result.get("error") + return report + for metric in result["metrics"]: + section = metric["section"] + head_id = ( + metric["id"].split(".", 1)[1] if section in PARTITIONED_SECTIONS else None + ) + for year, value in metric["values"].items(): + report["metrics"].append( + { + "metric_id": metric["id"], + "year": int(year), + "value": value, + "unit": metric["kind"], + "kind": KIND_BY_SECTION.get(section, section), + "partition": head_id, + "label": metric["label"], + } + ) + for check in result["checks"]: + report["identities"].append( + { + "identity_id": check["id"], + "year": None, + "status": check["status"], + "severity": check["severity"], + "detail": "; ".join(check["details"]), + } + ) + return report + + +def compute_budget_metrics( + sim, + factory: Optional[Callable] = None, + *, + years: Optional[Sequence[int]] = None, + partition: Optional[Partition] = None, + spec: Optional[Spec] = None, + reforms: Sequence[dict] = (), + run_guard: bool = True, + label: str = "candidate", + provenance: Optional[dict] = None, +) -> dict: + """Measure ``sim`` (a fresh ``Microsimulation``) and return the report. + + ``factory(reform=None)`` builds another simulation on the same data; by + default it rebuilds ``type(sim)`` from ``sim.dataset``. A failed identity + has status ``fail``; ``warn`` rows (a head that is zero everywhere) are + observations, not failures. + """ + spec = _spec_for_years(spec or load_spec(DEFAULT_SPEC_PATH), years) + if partition is None: + path = default_partition_path() + if path is None: + raise FileNotFoundError( + "the measured policyengine_uk ships no fiscal partition" + ) + partition = load_partition(path) + if factory is None: + dataset = sim.dataset + simulation_class = type(sim) + + def factory(reform=None): + return simulation_class(dataset=dataset, reform=reform or None) + + provenance = dict(provenance or {}) + provenance.setdefault("metrics_version", f"{TOOL_SCHEMA}+{tool_digest()[:16]}") + result, _ = measure( + sim, + factory, + partition, + spec, + label=label, + reforms=reforms, + run_guard=run_guard, + provenance=provenance, + ) + return to_report(result) diff --git a/policyengine_uk/utils/aggregate_impact/compare.py b/policyengine_uk/utils/aggregate_impact/compare.py new file mode 100644 index 0000000000..d8ef4a5ef9 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/compare.py @@ -0,0 +1,376 @@ +"""Compare a base and a head measurement. + +The comparison fails when the head did not run, ran on different data, or +broke a check the base passes; it reports, without failing, checks both sides +fail (pre-existing) and moves above the flag thresholds. +""" + +from __future__ import annotations + +import math +from typing import Mapping, Optional + +import numpy as np + +from . import TOOL_SCHEMA +from .spec import Spec, fiscal_year_label + +# Failing these on the head blocks the change whatever the base did: a +# non-finite or crashed result cannot be compared at all. +ALWAYS_BLOCKING_PREFIXES = ("finite:", "weights:") + +FLAG_KIND_BY_SECTION = { + "reforms": "reform_gbp", +} + +DECILES = tuple(range(1, 11)) + + +def _metric_map(result: Optional[Mapping]) -> dict: + if not result or result.get("status") != "ok": + return {} + return {m["id"]: m for m in result.get("metrics", [])} + + +def _check_map(result: Optional[Mapping]) -> dict: + if not result or result.get("status") != "ok": + return {} + return {c["id"]: c for c in result.get("checks", [])} + + +def _finite(value) -> bool: + return ( + value is not None and isinstance(value, (int, float)) and math.isfinite(value) + ) + + +def classify_check( + base_status: Optional[str], head_status: Optional[str], blocking: bool +) -> str: + """How a check's outcome on the head relates to the base. + + Returns one of: ``pass``, ``regression``, ``pre_existing``, ``fixed``, + ``new_warning``, ``warning``, ``removed``. + """ + if head_status is None: + return "removed" + if head_status == "pass": + return "fixed" if base_status in ("fail", "warn") else "pass" + if head_status == "warn": + return "warning" if base_status == "warn" else "new_warning" + # head failed + if blocking or base_status in (None, "pass"): + return "regression" + if base_status == "warn": + return "regression" + return "pre_existing" + + +def compare_metrics(base: Mapping, head: Mapping, spec: Spec, years) -> tuple: + rows, flags = [], [] + ids = list(dict.fromkeys(list(head) + list(base))) + for metric_id in ids: + head_metric, base_metric = head.get(metric_id), base.get(metric_id) + template = head_metric or base_metric + row = { + "id": metric_id, + "label": template["label"], + "section": template["section"], + "group": template.get("group"), + "kind": template["kind"], + "unit": template.get("unit"), + "values": {}, + "changed": False, + "only_in": None + if head_metric and base_metric + else ("head" if head_metric else "base"), + } + kind = FLAG_KIND_BY_SECTION.get(template["section"], template["kind"]) + threshold = spec.threshold(kind) + for year in years: + key = str(year) + b = base_metric["values"].get(key) if base_metric else None + h = head_metric["values"].get(key) if head_metric else None + delta = h - b if _finite(b) and _finite(h) else None + row["values"][key] = {"base": b, "head": h, "delta": delta} + if delta is not None and delta != 0: + row["changed"] = True + if delta is not None and threshold is not None and threshold.exceeded(b, h): + flags.append( + { + "id": metric_id, + "label": template["label"], + "section": template["section"], + "kind": template["kind"], + "year": year, + "base": b, + "head": h, + "delta": delta, + "limit": threshold.limit(b), + } + ) + rows.append(row) + return rows, flags + + +def _match(base_ids: np.ndarray, head_ids: np.ndarray) -> Optional[np.ndarray]: + """Indices into head arrays that line up with base households, or None.""" + if base_ids.shape == head_ids.shape and np.array_equal(base_ids, head_ids): + return np.arange(len(head_ids)) + order = np.argsort(head_ids, kind="stable") + positions = np.searchsorted(head_ids[order], base_ids) + positions = np.clip(positions, 0, len(order) - 1) + matched = order[positions] + if not np.array_equal(head_ids[matched], base_ids): + return None + return matched + + +def decile_impacts(base_households: Mapping, head_households: Mapping, years) -> tuple: + """Household-matched income changes within the base run's deciles. + + For each year and decile d of the base run: the mean change in the income + variable per household and the change relative to the decile's base + income, both weighted by the base household weights; plus the shares of + people (household weight times household size) whose household gains or + loses more than £1 a year. + """ + impacts, notes = {}, [] + for year in years: + key = str(year) + b, h = base_households.get(key), head_households.get(key) + if b is None or h is None: + notes.append(f"{fiscal_year_label(year)}: household detail missing") + continue + matched = _match(np.asarray(b["household_id"]), np.asarray(h["household_id"])) + if matched is None: + notes.append(f"{fiscal_year_label(year)}: base and head households differ") + continue + weight = np.asarray(b["household_weight"], dtype=np.float64) + head_weight = np.asarray(h["household_weight"], dtype=np.float64)[matched] + if not np.array_equal(weight, head_weight): + notes.append( + f"{fiscal_year_label(year)}: household weights changed; decile " + "changes use the base weights" + ) + people = np.asarray(b["household_count_people"], dtype=np.float64) + decile = np.asarray(b["decile"]) + base_income = np.asarray(b["income"], dtype=np.float64) + change = np.asarray(h["income"], dtype=np.float64)[matched] - base_income + rows = [] + for d in DECILES + ("all",): + mask = np.ones_like(decile, dtype=bool) if d == "all" else decile == d + w = weight[mask] + total_weight = float(w.sum()) + base_total = float(np.sum(w * base_income[mask])) + change_total = float(np.sum(w * change[mask])) + person_weight = w * people[mask] + person_total = float(person_weight.sum()) + rows.append( + { + "decile": d, + "households": total_weight, + "base_mean": base_total / total_weight + if total_weight + else float("nan"), + "mean_change": change_total / total_weight + if total_weight + else float("nan"), + "relative_change": change_total / base_total + if base_total + else float("nan"), + "share_gaining": float(np.sum(person_weight * (change[mask] > 1))) + / person_total + if person_total + else float("nan"), + "share_losing": float(np.sum(person_weight * (change[mask] < -1))) + / person_total + if person_total + else float("nan"), + } + ) + excluded = float(np.sum(weight[~np.isin(decile, DECILES)])) + if excluded: + notes.append( + f"{fiscal_year_label(year)}: {excluded:,.0f} households with negative " + "income have no decile and appear only in 'all'" + ) + impacts[key] = rows + return impacts, notes + + +def _runtime_differences(base: Mapping, head: Mapping) -> list: + def listing(result): + return { + line.split("==")[0]: line + for line in (result.get("runtime") or {}).get("distributions", []) + } + + b, h = listing(base), listing(head) + differences = [] + for name in sorted(set(b) | set(h)): + if name == "policyengine-uk": + continue + if b.get(name) != h.get(name): + differences.append( + f"{b.get(name, f'{name} absent')} -> {h.get(name, f'{name} absent')}" + ) + for field in ("python", "machine"): + bv, hv = ( + (base.get("runtime") or {}).get(field), + (head.get("runtime") or {}).get(field), + ) + if bv != hv: + differences.append(f"{field}: {bv} -> {hv}") + return differences + + +def compare( + base: Optional[Mapping], + head: Optional[Mapping], + spec: Spec, + base_households: Optional[Mapping] = None, + head_households: Optional[Mapping] = None, + strict: bool = False, +) -> dict: + failures, warnings = [], [] + head_ok = bool(head) and head.get("status") == "ok" + base_ok = bool(base) and base.get("status") == "ok" + if not head: + failures.append( + "The head measurement produced no result; see the measure job's log." + ) + elif not head_ok: + failures.append( + "The head measurement crashed: " + _last_line(head.get("error")) + ) + if not base: + warnings.append( + "The base measurement produced no result, so nothing is compared against it." + ) + elif not base_ok: + warnings.append( + "The base measurement crashed, so the head's own checks decide: " + + _last_line(base.get("error")) + ) + + years = (head or base or {}).get("years") or spec.years + if head_ok and base_ok: + if base.get("years") != head.get("years"): + failures.append( + f"Base and head measured different years ({base.get('years')} vs {head.get('years')})." + ) + b_sha = (base.get("provenance") or {}).get("dataset", {}).get("verified_sha256") + h_sha = (head.get("provenance") or {}).get("dataset", {}).get("verified_sha256") + if not b_sha or not h_sha or b_sha != h_sha: + failures.append( + f"Base and head did not run on the same verified dataset ({b_sha} vs {h_sha})." + ) + if (base.get("partition") or {}).get("sha256") != ( + head.get("partition") or {} + ).get("sha256"): + warnings.append( + "This change edits the fiscal partition; review the partition diff alongside the moves it explains." + ) + runtime_differences = _runtime_differences(base, head) + if runtime_differences: + warnings.append( + "Base and head ran on different runtimes, so some changes may come from dependencies: " + + "; ".join(runtime_differences[:10]) + ) + else: + runtime_differences = [] + + base_checks, head_checks = _check_map(base), _check_map(head) + checks = [] + for check_id in dict.fromkeys(list(head_checks) + list(base_checks)): + b, h = base_checks.get(check_id), head_checks.get(check_id) + blocking = check_id.startswith(ALWAYS_BLOCKING_PREFIXES) or not base_ok + classification = classify_check( + b["status"] if b else None, h["status"] if h else None, blocking + ) + if strict and classification == "pre_existing": + classification = "regression" + template = h or b + checks.append( + { + "id": check_id, + "label": template["label"], + "section": template.get("section", ""), + "base_status": b["status"] if b else None, + "head_status": h["status"] if h else None, + "classification": classification, + "head_details": (h or {}).get("details", []), + "base_details": (b or {}).get("details", []), + } + ) + regressions = [c for c in checks if c["classification"] == "regression"] + for check in regressions: + failures.append( + f"{check['label']} ({check['id']}): " + "; ".join(check["head_details"][:3]) + ) + + rows, flags = compare_metrics(_metric_map(base), _metric_map(head), spec, years) + for row in rows: + if row["only_in"] == "base" and head_ok: + warnings.append( + f"{row['label']} ({row['id']}) is measured on the base only." + ) + + deciles, decile_notes, decile_flags = {}, [], [] + if head_ok and base_ok and base_households and head_households: + deciles, decile_notes = decile_impacts(base_households, head_households, years) + threshold = spec.threshold("decile_relative") + for year, decile_rows in deciles.items(): + for row in decile_rows: + rel = row["relative_change"] + if ( + threshold is not None + and _finite(rel) + and abs(rel) > threshold.limit(0) + ): + decile_flags.append({"year": int(year), **row}) + + return { + "schema": TOOL_SCHEMA + "/comparison", + "status": "fail" if failures else "pass", + "failures": failures, + "warnings": warnings, + "years": years, + "checks": checks, + "rows": rows, + "flags": flags, + "deciles": deciles, + "decile_notes": decile_notes, + "decile_flags": decile_flags, + "runtime_differences": runtime_differences, + "base": _side_summary(base), + "head": _side_summary(head), + } + + +def _last_line(text) -> str: + lines = [line for line in str(text or "").strip().splitlines() if line.strip()] + return lines[-1] if lines else "no error recorded" + + +def _side_summary(result: Optional[Mapping]) -> Optional[dict]: + if not result: + return None + return { + "status": result.get("status"), + "label": result.get("label"), + "provenance": result.get("provenance"), + "runtime": { + k: v + for k, v in (result.get("runtime") or {}).items() + if k != "distributions" + }, + "partition": { + k: (result.get("partition") or {}).get(k) for k in ("source", "sha256") + }, + "guard": result.get("guard"), + "reforms": result.get("reforms"), + "timings": result.get("timings"), + "error": result.get("error"), + } diff --git a/policyengine_uk/utils/aggregate_impact/dataset.py b/policyengine_uk/utils/aggregate_impact/dataset.py new file mode 100644 index 0000000000..bdd32173bf --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/dataset.py @@ -0,0 +1,164 @@ +"""Pinned dataset selection: load a pin, fetch its file, verify its bytes.""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Optional, Union + +import yaml + +_HEX40 = re.compile(r"^[0-9a-f]{40}$") +_HEX64 = re.compile(r"^[0-9a-f]{64}$") +_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$") + + +class DatasetPinError(ValueError): + """The pin file is malformed or names a mutable revision.""" + + +class DatasetMismatchError(RuntimeError): + """A dataset file's bytes do not match the pinned SHA-256.""" + + +@dataclass(frozen=True) +class DatasetPin: + """An immutable reference to one dataset file on the Hugging Face Hub.""" + + repo_id: str + path: str + revision: str + sha256: str + data_package: str = "" + data_version: str = "" + hf_tag: Optional[str] = None + size_bytes: Optional[int] = None + certification: Mapping[str, str] = field(default_factory=dict) + + @property + def uri(self) -> str: + """The ``hf://`` reference ``policyengine_uk.Simulation`` accepts.""" + return f"hf://{self.repo_id}/{self.path}@{self.revision}" + + def problems(self) -> list[str]: + """Every reason this pin is not an immutable, verifiable reference.""" + problems = [] + if not _REPO_ID.match(self.repo_id or ""): + problems.append(f"repo_id {self.repo_id!r} is not 'owner/name'") + if not self.path or "@" in self.path or self.path.startswith("/"): + problems.append(f"path {self.path!r} is not a relative file path") + if not _HEX40.match(self.revision or ""): + problems.append( + f"revision {self.revision!r} is not a 40-character commit hash; " + "tags and branches can move" + ) + if not _HEX64.match(self.sha256 or ""): + problems.append(f"sha256 {self.sha256!r} is not a lowercase hex digest") + if self.size_bytes is not None and self.size_bytes <= 0: + problems.append(f"size_bytes {self.size_bytes!r} is not positive") + return problems + + def describe(self) -> str: + version = f"{self.data_package} {self.data_version}".strip() + return ( + f"{self.path} ({version}, commit {self.revision[:8]}, " + f"sha256 {self.sha256[:12]})" + ) + + def as_dict(self) -> dict: + return { + "uri": self.uri, + "repo_id": self.repo_id, + "path": self.path, + "revision": self.revision, + "sha256": self.sha256, + "data_package": self.data_package, + "data_version": self.data_version, + "hf_tag": self.hf_tag, + "size_bytes": self.size_bytes, + "certification": dict(self.certification), + } + + +def load_pin(path: Union[str, os.PathLike]) -> DatasetPin: + """Read a pin file and reject it unless it is immutable and complete.""" + with open(path) as f: + raw = yaml.safe_load(f) or {} + known = { + "repo_id", + "path", + "revision", + "sha256", + "data_package", + "data_version", + "hf_tag", + "size_bytes", + "certification", + } + unknown = sorted(set(raw) - known) + if unknown: + raise DatasetPinError(f"{path}: unknown keys {unknown}") + pin = DatasetPin( + repo_id=str(raw.get("repo_id", "")), + path=str(raw.get("path", "")), + revision=str(raw.get("revision", "")), + sha256=str(raw.get("sha256", "")), + data_package=str(raw.get("data_package", "")), + data_version=str(raw.get("data_version", "")), + hf_tag=None if raw.get("hf_tag") is None else str(raw["hf_tag"]), + size_bytes=None if raw.get("size_bytes") is None else int(raw["size_bytes"]), + certification={ + str(k): str(v) for k, v in (raw.get("certification") or {}).items() + }, + ) + problems = pin.problems() + if problems: + raise DatasetPinError(f"{path}: " + "; ".join(problems)) + return pin + + +def sha256_file(path: Union[str, os.PathLike], chunk_size: int = 1 << 20) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_file(path: Union[str, os.PathLike], expected_sha256: str) -> str: + """Return the file's SHA-256, raising if it differs from the expected one.""" + actual = sha256_file(path) + if actual != expected_sha256: + raise DatasetMismatchError( + f"{path} has sha256 {actual}, but the pin requires {expected_sha256}" + ) + return actual + + +def materialise( + pin: DatasetPin, + token: Optional[str] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, +) -> Path: + """Download the pinned file (or reuse the cached copy) and verify it. + + The token defaults to ``HUGGING_FACE_TOKEN``. The Hugging Face cache is + keyed by commit, so a verified file is never downloaded twice. + """ + from huggingface_hub import hf_hub_download + + if token is None: + token = os.environ.get("HUGGING_FACE_TOKEN") or None + local = hf_hub_download( + repo_id=pin.repo_id, + filename=pin.path, + revision=pin.revision, + repo_type="model", + token=token, + cache_dir=cache_dir, + ) + verify_file(local, pin.sha256) + return Path(local) diff --git a/policyengine_uk/utils/aggregate_impact/guard.py b/policyengine_uk/utils/aggregate_impact/guard.py new file mode 100644 index 0000000000..9850d06f9d --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/guard.py @@ -0,0 +1,276 @@ +"""The counted-once check: every component enters each total exactly once. + +For each component of a partition total, the check adds 1 to every unit +where the component is non-zero, recomputes the partition's heads, and +measures how much each total it belongs to moved per unit added (its +coefficient). Heads name one variable each, so a coefficient above 1 means +another head's variable contains the component as well: a double count of +the kind #2159 reports. On tax totals nothing should offset the addition, so +the coefficient must be exactly 1. On spending totals means tests can claw +the addition back (Pension Credit falls when State Pension rises), so the +coefficient may be below 1 but never above it: above 1 means either a double +count or another benefit rising when a household's income rises. Households +whose spending jumps far beyond the addition have crossed an eligibility +threshold; they are left out of the coefficient and counted instead. + +A declared double count (``model_double_counts``) is perturbed too: its +coefficient in its container must still be 1, or the declaration is stale. + +The check reuses one simulation. Between perturbations it discards every +formula result and keeps the supplied inputs, then restores any input it +overrode; at the end it recomputes the baseline and requires it to match the +first computation exactly. +""" + +from __future__ import annotations + +from typing import Iterable, Mapping, Optional + +import numpy as np + +from .partition import Partition + +# On spending totals, a household's change counts as an eligibility change +# (and is left out of the coefficient) when it differs from what was added by +# more than this many pounds, or this many times what was added. +ELIGIBILITY_JUMP_GBP = 50.0 +ELIGIBILITY_JUMP_FACTOR = 20.0 + + +def _raw(sim, variable, period): + try: + result = sim.calculate(variable, period, unweighted=True) + except TypeError: + result = sim.calculate(variable, period) + return np.asarray(getattr(result, "values", result)) + + +def snapshot_inputs(sim, variables) -> dict: + """Copy the supplied input arrays of ``variables`` before any calculation.""" + snapshot = {} + for variable in variables: + if variable not in sim.tax_benefit_system.variables: + continue + holder = sim.get_holder(variable) + periods = holder.get_known_periods() + if periods: + snapshot[variable] = [ + (period, np.array(holder.get_array(period), copy=True)) + for period in periods + ] + return snapshot + + +def _restore(sim, variable, snapshot): + sim.delete_arrays(variable) + for period, values in snapshot.get(variable, []): + sim.set_input(variable, period, values) + + +def _months(year): + from policyengine_core.periods import period as make_period + + return make_period(str(year)).get_subperiods("month") + + +def _household(sim, values, entity): + values = np.asarray(values, dtype=np.float64) + if entity == "household": + return values + return np.asarray(sim.map_result(values, entity, "household"), dtype=np.float64) + + +def _totals_hh(sim, partition, year, totals, active=()): + out = {} + for total in totals: + hh = None + for head in partition.heads_in(total, active): + values = _raw(sim, head.variable, year) + entity = sim.tax_benefit_system.variables[head.variable].entity.key + mapped = _household(sim, values, entity) + hh = mapped if hh is None else hh + mapped + out[total] = hh + return out + + +def counted_once( + sim, + partition: Partition, + year: int, + tolerance: float, + snapshot: Optional[Mapping] = None, + active: Iterable[str] = (), + variables_to_check: Optional[Iterable[str]] = None, +) -> dict: + """Perturb each component (or those in ``variables_to_check``) once.""" + reset = getattr(sim, "_invalidate_all_caches", None) + if reset is None: + return { + "status": "skipped", + "reason": "this policyengine-core has no way to discard formula results", + } + snapshot = snapshot or {} + variables = sim.tax_benefit_system.variables + model_totals = [t for t in partition.totals if t in variables] + partition_totals = [t for t in partition.totals if partition.heads_in(t)] + + reset() + hw = _raw(sim, "household_weight", year).astype(np.float64) + heads0 = _totals_hh(sim, partition, year, partition_totals, active) + model0 = {t: _raw(sim, t, year).astype(np.float64) for t in model_totals} + containers = {dc.contained_in for dc in partition.double_counts} + container0 = { + c: _household(sim, _raw(sim, c, year), variables[c].entity.key) + for c in containers + } + baseline = {} + for variable in partition.component_variables(): + definition = variables[variable] + if definition.definition_period == "month": + baseline[variable] = [ + (month, _raw(sim, variable, month).astype(np.float64)) + for month in _months(year) + ] + else: + baseline[variable] = _raw(sim, variable, year).astype(np.float64) + + results, double_count_results = [], [] + declared = {dc.variable: dc for dc in partition.double_counts} + selected = None if variables_to_check is None else set(variables_to_check) + for variable in partition.component_variables(): + if selected is not None and variable not in selected: + continue + definition = variables[variable] + entity = definition.entity.key + totals = [ + t for t in partition_totals if variable in partition.components(t, active) + ] + tax_totals = [t for t in totals if partition.is_tax_total(t)] + record = { + "variable": variable, + "totals": totals, + "coefficients": {}, + "model_coefficients": {}, + "units": 0, + "status": "pass", + "detail": "", + } + if definition.value_type is not float: + record.update(status="skipped", detail="not a money variable") + results.append(record) + continue + if definition.definition_period == "month": + year_values = sum(values for _, values in baseline[variable]) + else: + year_values = baseline[variable] + nonzero = year_values != 0 + if nonzero.any(): + mask = nonzero + elif tax_totals: + # A tax that is zero everywhere can still be tested: taxes don't + # passport anyone to anything. + mask = np.ones_like(nonzero) + else: + record.update( + status="skipped", + detail=( + "zero for every unit; adding to it would create recipients, " + "whom passporting rules could treat differently" + ), + ) + results.append(record) + continue + added = mask.astype(np.float64) + expected = _household(sim, added, entity) + expected_total = float(np.sum(hw * expected)) + record["units"] = int(mask.sum()) + + reset() + if definition.definition_period == "month": + for month, values in baseline[variable]: + sim.set_input(variable, month, values + added / 12) + else: + sim.set_input(variable, year, baseline[variable] + added) + heads1 = _totals_hh(sim, partition, year, totals, active) + problems = [] + record["eligibility_changes"] = {} + for total in totals: + moved = heads1[total] - heads0[total] + if total in tax_totals: + k = float(np.sum(hw * moved)) / expected_total + if abs(k - 1) > tolerance: + problems.append(f"{total} moves by {k:.4f} per unit, not 1") + else: + # A household whose spending jumps by far more than it was + # given has crossed an eligibility threshold rather than + # counted the component twice, which adds exactly one more + # per unit. Leave those households out, and count them. + jumped = np.abs(moved - expected) > np.maximum( + ELIGIBILITY_JUMP_GBP, ELIGIBILITY_JUMP_FACTOR * expected + ) + kept = float(np.sum(hw * expected * ~jumped)) + record["eligibility_changes"][total] = int(jumped.sum()) + if kept <= 0: + k = float("nan") + problems.append( + f"{total}: every perturbed household changed eligibility" + ) + else: + k = float(np.sum(hw * moved * ~jumped)) / kept + if k > 1 + tolerance: + problems.append( + f"{total} moves by {k:.4f} per unit, more than 1" + ) + record["coefficients"][total] = k + for total in model_totals: + if variable in partition.components(total, active): + moved = _raw(sim, total, year).astype(np.float64) - model0[total] + record["model_coefficients"][total] = ( + float(np.sum(hw * moved)) / expected_total + ) + if variable in declared: + dc = declared[variable] + moved = ( + _household( + sim, + _raw(sim, dc.contained_in, year), + variables[dc.contained_in].entity.key, + ) + - container0[dc.contained_in] + ) + k = float(np.sum(hw * moved)) / expected_total + present = abs(k - 1) <= tolerance + double_count_results.append( + { + "variable": variable, + "contained_in": dc.contained_in, + "coefficient": k, + "issue": dc.issue, + "status": "pass" if present else "fail", + "detail": "" + if present + else ( + f"{dc.contained_in} moves by {k:.4f} per unit of {variable}: " + "the model no longer counts it twice, so remove the " + "declaration and give it its own head" + ), + } + ) + if problems: + record.update(status="fail", detail="; ".join(problems)) + results.append(record) + _restore(sim, variable, snapshot) + + reset() + heads_again = _totals_hh(sim, partition, year, partition_totals, active) + reproduces = all( + np.array_equal(heads_again[t], heads0[t]) for t in partition_totals + ) + return { + "status": "ran", + "year": year, + "tolerance": tolerance, + "components": results, + "double_counts": double_count_results, + "reset_reproduces_baseline": bool(reproduces), + } diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py new file mode 100644 index 0000000000..4f4080aecb --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -0,0 +1,765 @@ +"""Measure one model checkout on one dataset over the report horizon.""" + +from __future__ import annotations + +import gc +import hashlib +import json +import platform +import re +import sys +import time +from importlib import metadata +from pathlib import Path +from typing import Callable, Mapping, Optional, Sequence + +import numpy as np + +from . import TOOL_SCHEMA +from .guard import counted_once, snapshot_inputs +from .partition import Partition, closure_problems, group_label, read_model_lists +from .spec import Spec, fiscal_year_label + +ENTITIES = ("person", "benunit", "household") + +# Model totals reported alongside the partition. Each is a household variable. +HOUSEHOLD_TOTALS = { + "household_market_income": "Household market income", + "household_net_income": "Household net income", + "hbai_household_net_income": "Household net income (HBAI definition)", +} + + +class Recorder: + """Collects metric values and check outcomes for one measurement.""" + + def __init__(self): + self.metrics: dict = {} + self.checks: dict = {} + + def metric( + self, metric_id, label, section, kind, year, value, group=None, unit=None + ): + record = self.metrics.setdefault( + metric_id, + { + "id": metric_id, + "label": label, + "section": section, + "group": group, + "kind": kind, + "unit": unit, + "values": {}, + }, + ) + record["values"][str(year)] = float(value) + + def check(self, check_id, label, ok, detail="", severity="fail", section=""): + """Record a check; repeated ids (one per year) merge into one result.""" + status = "pass" if ok else ("warn" if severity == "warn" else "fail") + record = self.checks.get(check_id) + if record is None: + record = { + "id": check_id, + "label": label, + "section": section, + "severity": severity, + "status": status, + "details": [], + } + self.checks[check_id] = record + elif status != "pass" and record["status"] == "pass": + record["status"] = status + if not ok and detail: + record["details"].append(detail) + + +def array(sim, variable: str, period, map_to: Optional[str] = None) -> np.ndarray: + """A variable's raw values, without survey weights.""" + try: + result = sim.calculate(variable, period, map_to=map_to, unweighted=True) + except TypeError: + result = sim.calculate(variable, period, map_to=map_to) + return np.asarray(getattr(result, "values", result)) + + +def active_parameters(sim, partition: Partition, year) -> set: + """The ``dropped_when`` switches that are on in ``year``.""" + active = set() + for path in partition.dropped_when_parameters(): + node = sim.tax_benefit_system.parameters + try: + for part in path.split("."): + node = getattr(node, part) + if bool(node(f"{year}-01-01")): + active.add(path) + except AttributeError: + continue + return active + + +def entity_of(sim, variable: str) -> str: + return sim.tax_benefit_system.variables[variable].entity.key + + +def to_household(sim, values: np.ndarray, entity: str) -> np.ndarray: + if entity == "household": + return np.asarray(values, dtype=np.float64) + return np.asarray( + sim.map_result(np.asarray(values, dtype=np.float64), entity, "household"), + dtype=np.float64, + ) + + +def weighted_gini(values: np.ndarray, weights: np.ndarray) -> float: + """Gini coefficient of ``values`` with non-negative ``weights``.""" + values = np.asarray(values, dtype=np.float64) + weights = np.asarray(weights, dtype=np.float64) + keep = weights > 0 + values, weights = values[keep], weights[keep] + if values.size == 0: + return float("nan") + order = np.argsort(values, kind="stable") + values, weights = values[order], weights[order] + cumulative_weight = np.cumsum(weights) + cumulative_income = np.cumsum(weights * values) + total_weight, total_income = cumulative_weight[-1], cumulative_income[-1] + if total_income == 0: + return float("nan") + previous = np.concatenate([[0.0], cumulative_income[:-1]]) + area = np.sum(weights * (cumulative_income + previous)) / ( + total_weight * total_income + ) + return float(1.0 - area) + + +def slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def tool_digest() -> str: + """SHA-256 of this package's code and spec, independent of where it lives.""" + root = Path(__file__).resolve().parent + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.suffix in (".py", ".yaml") and "__pycache__" not in path.parts: + digest.update(path.relative_to(root).as_posix().encode() + b"\0") + digest.update(path.read_bytes() + b"\0") + return digest.hexdigest() + + +def runtime_identity() -> dict: + """The installed distributions and interpreter that ran the measurement.""" + distributions = sorted( + { + (d.metadata["Name"].lower().replace("_", "-"), d.version) + for d in metadata.distributions() + if d.metadata["Name"] + } + ) + listing = "\n".join(f"{name}=={version}" for name, version in distributions) + + def version(name): + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + return { + "python": sys.version.split()[0], + "platform": platform.platform(), + "machine": platform.machine(), + "policyengine_uk": version("policyengine-uk"), + "policyengine_core": version("policyengine-core"), + "numpy": version("numpy"), + "distributions": [f"{n}=={v}" for n, v in distributions], + "distributions_sha256": hashlib.sha256(listing.encode()).hexdigest(), + } + + +def _identity_check(rec, check_id, label, components_hh, total_hh, year, spec): + """Household-level accounting identity: components add up to the total.""" + difference = components_hh - total_hh + scale = np.abs(total_hh) + 1.0 + tolerance = 1.0 + spec.identity_relative_tolerance * scale + bad = np.abs(difference) > tolerance + finite = np.all(np.isfinite(difference)) + ok = bool(finite and not bad.any()) + detail = "" + if not ok: + worst = int(np.nanargmax(np.abs(difference))) if difference.size else 0 + detail = ( + f"{fiscal_year_label(year)}: {int(bad.sum())} households differ; " + f"largest gap £{float(difference[worst]):,.2f}" + ) + rec.check(check_id, label, ok, detail, section="identities") + + +def measure_year( + sim, partition: Partition, spec: Spec, year: int, rec: Recorder, households: dict +): + tbs_variables = sim.tax_benefit_system.variables + weights = {e: array(sim, f"{e}_weight", year).astype(np.float64) for e in ENTITIES} + for entity, label in ( + ("person", "People"), + ("benunit", "Benefit units"), + ("household", "Households"), + ): + w = weights[entity] + rec.check( + f"weights:{entity}", + f"{label} weights are finite and non-negative", + bool(np.all(np.isfinite(w)) and np.all(w >= 0)), + f"{fiscal_year_label(year)}: invalid {entity} weights", + section="data", + ) + rec.metric(f"population.{entity}", label, "population", "count", year, w.sum()) + hw = weights["household"] + + head_hh = {} + for head in partition.heads: + entity = entity_of(sim, head.variable) + values = array(sim, head.variable, year).astype(np.float64) + finite = np.isfinite(values) + rec.check( + f"finite:{head.id}", + f"{head.label} is finite for every unit", + bool(finite.all()), + f"{fiscal_year_label(year)}: {int((~finite).sum())} non-finite values", + section="finite", + ) + hh = to_household(sim, values, entity) + head_hh[head.variable] = hh + section = "receipts" if head.kind == "tax" else "spending" + rec.metric( + f"{section}.{head.id}", + head.label, + section, + "gbp", + year, + float(np.sum(hw * hh)), + group=group_label(head.report_group), + ) + if head.kind == "spending" and head.caseload: + rec.metric( + f"caseloads.{head.id}", + head.label, + "caseloads", + "count", + year, + float(np.sum(weights[entity] * (values > 0))), + group=group_label(head.report_group), + unit=entity, + ) + if head.sign == "nonnegative": + negative = values < -0.005 + rec.check( + f"sign:{head.id}", + f"{head.label} is never negative", + not bool(negative.any()), + f"{fiscal_year_label(year)}: {int(negative.sum())} units below zero, " + f"minimum £{float(np.nanmin(values)):,.2f}", + section="signs", + ) + if not head.zero_at_baseline: + rec.check( + f"inert:{head.id}", + f"{head.label} is not zero for every unit", + bool(np.any(values != 0)), + f"{fiscal_year_label(year)}: zero for every unit", + severity="warn", + section="coverage", + ) + + double_count_hh = {} + for dc in partition.double_counts: + values = array(sim, dc.variable, year).astype(np.float64) + hh = to_household(sim, values, entity_of(sim, dc.variable)) + double_count_hh[dc.variable] = hh + rec.metric( + f"double_counts.{dc.variable}", + f"{dc.variable}, counted again inside {dc.contained_in}", + "double_counts", + "gbp", + year, + float(np.sum(hw * hh)), + ) + + active = active_parameters(sim, partition, year) + model_hh = {} + for total in partition.totals: + if total not in tbs_variables: + continue + hh = to_household( + sim, array(sim, total, year).astype(np.float64), entity_of(sim, total) + ) + model_hh[total] = hh + rec.metric( + f"model.{total}", total, "model_totals", "gbp", year, float(np.sum(hw * hh)) + ) + components = np.zeros_like(hh) + for head in partition.heads_in(total, active): + components = components + head_hh[head.variable] + for dc in partition.double_counts_in(total, active): + components = components + double_count_hh[dc.variable] + _identity_check( + rec, + f"identity:{total}", + f"The partition's heads and declared double counts add up to {total}", + components, + hh, + year, + spec, + ) + + def partition_sum(total): + out = np.zeros_like(hw) + for head in partition.heads_in(total, active): + out = out + head_hh[head.variable] + return out + + receipts = partition_sum("gov_tax") + spending = partition_sum("gov_spending") + rec.metric( + "balance.receipts", + "Government receipts, each component once", + "balance", + "gbp", + year, + float(np.sum(hw * receipts)), + ) + rec.metric( + "balance.spending", + "Government spending", + "balance", + "gbp", + year, + float(np.sum(hw * spending)), + ) + rec.metric( + "balance.balance", + "Government balance, each receipt once", + "balance", + "gbp", + year, + float(np.sum(hw * (receipts - spending))), + ) + household_only = np.zeros_like(hw) + for head in partition.spending_heads: + if "household_benefits" in head.totals and "gov_spending" not in head.totals: + household_only = household_only + head_hh[head.variable] + rec.metric( + "balance.household_only_benefits", + "Benefits in household_benefits but not in gov_spending", + "balance", + "gbp", + year, + float(np.sum(hw * household_only)), + ) + + if {"gov_balance", "gov_tax", "gov_spending"} <= set(tbs_variables): + gov_balance = array(sim, "gov_balance", year).astype(np.float64) + rec.metric( + "model.gov_balance", + "gov_balance", + "model_totals", + "gbp", + year, + float(np.sum(hw * gov_balance)), + ) + _identity_check( + rec, + "identity:gov_balance", + "gov_balance equals gov_tax minus gov_spending", + model_hh.get("gov_tax", array(sim, "gov_tax", year)) + - model_hh.get("gov_spending", array(sim, "gov_spending", year)), + gov_balance, + year, + spec, + ) + for variable, label in HOUSEHOLD_TOTALS.items(): + if variable in tbs_variables: + values = array(sim, variable, year).astype(np.float64) + rec.metric( + f"household.{variable}", + label, + "household_totals", + "gbp", + year, + float(np.sum(hw * values)), + ) + if { + "household_net_income", + "household_market_income", + "household_benefits", + "household_tax", + "pension_contributions", + } <= set(tbs_variables): + pension = to_household( + sim, + array(sim, "pension_contributions", year).astype(np.float64), + entity_of(sim, "pension_contributions"), + ) + components = ( + array(sim, "household_market_income", year).astype(np.float64) + + model_hh.get("household_benefits", array(sim, "household_benefits", year)) + - model_hh.get("household_tax", array(sim, "household_tax", year)) + - pension + ) + _identity_check( + rec, + "identity:household_net_income", + "Household net income equals market income plus benefits less taxes and pension contributions", + components, + array(sim, "household_net_income", year).astype(np.float64), + year, + spec, + ) + + pw = weights["person"] + for measure_spec in spec.poverty_measures: + try: + in_poverty = array( + sim, measure_spec["variable"], year, map_to="person" + ).astype(bool) + except Exception as error: # noqa: BLE001 - recorded, not raised + rec.check( + f"available:{measure_spec['id']}", + f"{measure_spec['label']} can be computed", + False, + f"{fiscal_year_label(year)}: {type(error).__name__}: {error}", + section="coverage", + ) + continue + for group in spec.poverty_groups: + if group.get("variable"): + try: + members = array(sim, group["variable"], year).astype(bool) + except Exception as error: # noqa: BLE001 + rec.check( + f"available:{group['id']}", + f"{group['label']} can be identified", + False, + f"{fiscal_year_label(year)}: {type(error).__name__}: {error}", + section="coverage", + ) + continue + else: + members = np.ones_like(in_poverty, dtype=bool) + population = float(np.sum(pw * members)) + poor = float(np.sum(pw * (members & in_poverty))) + rate = poor / population if population > 0 else float("nan") + rec.check( + "bounds:poverty_rates", + "Poverty rates lie between 0 and 1", + bool(0.0 <= rate <= 1.0), + f"{fiscal_year_label(year)}: {measure_spec['id']} {group['id']} is {rate}", + section="bounds", + ) + metric_id = f"{measure_spec['id']}.{group['id']}" + label = f"{measure_spec['label']}: {group['label'].lower()}" + rec.metric( + f"poverty.{metric_id}", + label, + "poverty", + "rate", + year, + rate, + group=measure_spec["label"], + ) + rec.metric( + f"poverty_count.{metric_id}", + label, + "poverty_count", + "count", + year, + poor, + group=measure_spec["label"], + ) + + people = array(sim, "household_count_people", year).astype(np.float64) + for inequality in spec.inequality: + income = array(sim, inequality["variable"], year).astype(np.float64) + rec.metric( + f"inequality.{inequality['id']}", + inequality["label"], + "inequality", + "index", + year, + weighted_gini(income, hw * people), + ) + + decile = array(sim, spec.decile_variable, year).astype(np.int64) + income = array(sim, spec.income_variable, year).astype(np.float64) + for d in range(1, 11): + in_decile = decile == d + weight = float(np.sum(hw[in_decile])) + rec.metric( + f"deciles.mean_income.{d}", + f"Decile {d}: mean {spec.income_variable} per household", + "deciles", + "gbp_household", + year, + float(np.sum(hw[in_decile] * income[in_decile])) / weight + if weight + else float("nan"), + group="Own-decile levels", + ) + households[str(year)] = { + "household_id": array(sim, "household_id", year).astype(np.int64), + "household_weight": hw.astype(np.float64), + "household_count_people": people.astype(np.float64), + "decile": decile, + "income": income, + } + + +def _finite_metrics_check(rec: Recorder): + bad = [ + f"{m['id']} {fiscal_year_label(int(year))}" + for m in rec.metrics.values() + for year, value in m["values"].items() + if not np.isfinite(value) + ] + rec.check( + "finite:metrics", + "Every reported figure is finite", + not bad, + "; ".join(bad[:20]) + (f" and {len(bad) - 20} more" if len(bad) > 20 else ""), + section="finite", + ) + + +def measure_reforms( + factory: Callable, + baseline_balance: Mapping[int, float], + reforms: Sequence[Mapping], + years: Sequence[int], + rec: Recorder, +) -> list: + """Each reform's change in gov_balance against the baseline run, by year. + + First rebuilds an unreformed simulation the same way, which must + reproduce the baseline exactly: otherwise reform costings would include + differences that no reform caused. + """ + outcomes = [] + first = years[0] + try: + fresh = factory(reform=None) + hw = array(fresh, "household_weight", first).astype(np.float64) + again = float( + np.sum(hw * array(fresh, "gov_balance", first).astype(np.float64)) + ) + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + again == baseline_balance[first], + f"{fiscal_year_label(first)}: £{again:,.0f} against £{baseline_balance[first]:,.0f}", + section="determinism", + ) + del fresh + except Exception as error: # noqa: BLE001 + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + False, + f"{type(error).__name__}: {error}", + section="determinism", + ) + gc.collect() + for reform in reforms: + name = str(reform["name"]) + reform_id = slug(name) + outcome = { + "id": reform_id, + "name": name, + "parameters": reform.get("parameters", {}), + "error": None, + } + try: + reformed = factory(reform=reform.get("parameters")) + for year in years: + hw = array(reformed, "household_weight", year).astype(np.float64) + balance = float( + np.sum(hw * array(reformed, "gov_balance", year).astype(np.float64)) + ) + rec.metric( + f"reforms.{reform_id}", + name, + "reforms", + "gbp", + year, + balance - baseline_balance[year], + ) + del reformed + except Exception as error: # noqa: BLE001 - one reform's failure is reported, not fatal + outcome["error"] = f"{type(error).__name__}: {error}" + gc.collect() + outcomes.append(outcome) + return outcomes + + +def measure( + sim, + factory: Callable, + partition: Partition, + spec: Spec, + *, + label: str, + reforms: Sequence[Mapping] = (), + run_guard: bool = True, + provenance: Optional[Mapping] = None, + guard_variables: Optional[Sequence[str]] = None, +) -> tuple: + """Measure a simulation over the spec's horizon. + + ``sim`` is a fresh simulation on the dataset; ``factory(reform=None)`` + builds another on the same dataset. Returns ``(result, households)``. + """ + started = time.time() + timings = {} + rec = Recorder() + model_lists, list_problems = read_model_lists(partition) + problems = list_problems + closure_problems( + partition, model_lists, sim.tax_benefit_system.variables + ) + rec.check( + "closure", + "Each model total's components are exactly the partition's heads and declared double counts", + not problems, + "; ".join(problems), + section="partition", + ) + snapshot = snapshot_inputs(sim, partition.component_variables()) + + households = {} + baseline_balance = {} + reset = getattr(sim, "_invalidate_all_caches", None) + for year in spec.years: + t = time.time() + measure_year(sim, partition, spec, year, rec, households) + hw = array(sim, "household_weight", year).astype(np.float64) + baseline_balance[year] = float( + np.sum(hw * array(sim, "gov_balance", year).astype(np.float64)) + ) + timings[f"year_{year}"] = round(time.time() - t, 2) + if reset is not None: + # Keep memory flat across the horizon; supplied inputs survive. + reset() + gc.collect() + _finite_metrics_check(rec) + + guard = {"status": "skipped", "reason": "not requested"} + if run_guard: + t = time.time() + guard = counted_once( + sim, + partition, + spec.guard_year, + spec.counted_once_tolerance, + snapshot, + active=active_parameters(sim, partition, spec.guard_year), + variables_to_check=guard_variables, + ) + timings["counted_once"] = round(time.time() - t, 2) + for result in guard.get("components", []): + if result["status"] == "skipped": + continue + rec.check( + f"counted_once:{result['variable']}", + f"Adding £1 to {result['variable']} adds exactly £1 to each tax total it is in, " + "and at most £1 to each spending total", + result["status"] == "pass", + result["detail"], + section="counted_once", + ) + for result in guard.get("double_counts", []): + rec.check( + f"double_count_present:{result['variable']}", + f"{result['variable']} is still contained in {result['contained_in']}", + result["status"] == "pass", + result["detail"], + section="counted_once", + ) + if "reset_reproduces_baseline" in guard: + rec.check( + "counted_once:reset_reproduces_baseline", + "The counted-once check leaves the baseline unchanged", + bool(guard["reset_reproduces_baseline"]), + "recomputed baseline totals differ from the first run", + section="counted_once", + ) + if guard.get("status") == "skipped": + rec.check( + "counted_once:ran", + "The counted-once check ran", + False, + guard.get("reason", ""), + severity="warn", + section="counted_once", + ) + + t = time.time() + reform_outcomes = measure_reforms( + factory, baseline_balance, reforms, spec.years, rec + ) + timings["reforms"] = round(time.time() - t, 2) + for outcome in reform_outcomes: + rec.check( + f"reform_runs:{outcome['id']}", + f"The reform '{outcome['name']}' runs", + outcome["error"] is None, + outcome["error"] or "", + section="reforms", + ) + timings["total"] = round(time.time() - started, 2) + + result = { + "schema": TOOL_SCHEMA, + "status": "ok", + "label": label, + "years": spec.years, + "provenance": dict(provenance or {}), + "runtime": runtime_identity(), + "partition": partition.as_dict(), + "metrics": list(rec.metrics.values()), + "checks": list(rec.checks.values()), + "guard": guard, + "reforms": reform_outcomes, + "timings": timings, + } + return result, households + + +def write_result(out_dir, result: Mapping, households: Optional[Mapping] = None): + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + with open(out_dir / "result.json", "w") as f: + json.dump(result, f, indent=1, sort_keys=False, allow_nan=True) + if households: + arrays = { + f"{year}__{name}": values + for year, columns in households.items() + for name, values in columns.items() + } + np.savez_compressed(out_dir / "households.npz", **arrays) + + +def read_result(out_dir) -> tuple: + """Load ``(result, households)`` written by :func:`write_result`.""" + out_dir = Path(out_dir) + path = out_dir / "result.json" + if not path.exists(): + return None, None + with open(path) as f: + result = json.load(f) + households = None + npz = out_dir / "households.npz" + if npz.exists(): + households = {} + with np.load(npz) as data: + for key in data.files: + year, _, name = key.partition("__") + households.setdefault(year, {})[name] = data[key] + return result, households diff --git a/policyengine_uk/utils/aggregate_impact/partition.py b/policyengine_uk/utils/aggregate_impact/partition.py new file mode 100644 index 0000000000..a853270b53 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/partition.py @@ -0,0 +1,545 @@ +"""The fiscal partition: loading, structural validation and closure checks. + +The partition lives with the model at +``policyengine_uk/data/fiscal_partition.yaml`` (schema 1; its header defines +the fields). This module reads the YAML directly, not through any loader in +the model, so the pull request head's copy of this tool can read the base +checkout's partition. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Mapping, Optional + +import yaml + +SUPPORTED_SCHEMA_VERSIONS = {1} +PARTITION_RELATIVE_PATH = Path("data") / "fiscal_partition.yaml" +ENTITIES = {"person", "benunit", "household"} +SIGNS = {None, "nonnegative", "any"} + +# Where each total's component list lives in the model, and which kind of +# heads it divides into. A total in the file may override these. +KNOWN_TOTALS = { + "gov_tax": ("tax", "policyengine_uk.variables.gov.gov_tax", "GOV_TAX_VARIABLES"), + "household_tax": ( + "tax", + "policyengine_uk.variables.gov.hmrc.household_tax", + "HOUSEHOLD_TAX_VARIABLES", + ), + "pre_budget_change_household_tax": ( + "tax", + "policyengine_uk.variables.contrib.policyengine.pre_budget_change_household_tax", + "PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES", + ), + "gov_spending": ( + "spending", + "policyengine_uk.variables.gov.gov_spending", + "GOV_SPENDING_VARIABLES", + ), + "household_benefits": ( + "spending", + "policyengine_uk.variables.household.income.household_benefits", + "HOUSEHOLD_BENEFIT_VARIABLES", + ), +} +GOVERNMENT_TOTALS = {"gov_tax", "gov_spending"} + + +class PartitionError(ValueError): + """The partition file is malformed or internally inconsistent.""" + + +@dataclass(frozen=True) +class TotalSpec: + name: str + kind: str # "tax" or "spending" + module: str + list_name: str + label: str = "" + dropped_when: tuple = () # ((parameter path, (variable, ...)), ...) + + +@dataclass(frozen=True) +class Head: + id: str + label: str + variable: str + totals: tuple + report_group: str + kind: str # "tax" or "spending" + entity: Optional[str] = None + obr_line: Optional[str] = None + payer: Optional[str] = None + sign: Optional[str] = None + zero_at_baseline: bool = False + caseload: bool = False + reads: tuple = () + notes: tuple = () + + +@dataclass(frozen=True) +class DoubleCount: + variable: str + contained_in: str + totals: tuple + issue: str = "" + + +@dataclass(frozen=True) +class Exclusion: + variable: str + kind: str = "" + head: str = "" + spans: tuple = () + note: str = "" + + +@dataclass(frozen=True) +class Partition: + schema_version: int + totals: Mapping[str, TotalSpec] + tax_heads: tuple + spending_heads: tuple + double_counts: tuple + exclusions: tuple + overlap_guards: tuple = () + source: str = "" + sha256: str = "" + raw: Mapping = field(default_factory=dict, compare=False, repr=False) + + @property + def heads(self) -> tuple: + return self.tax_heads + self.spending_heads + + def dropped(self, total: str, active: Iterable[str] = ()) -> set: + """Variables the total leaves out while the given parameters are true.""" + active = set(active) + spec = self.totals.get(total) + if spec is None: + return set() + return { + v + for parameter, variables in spec.dropped_when + if parameter in active + for v in variables + } + + def heads_in(self, total: str, active: Iterable[str] = ()) -> list: + dropped = self.dropped(total, active) + return [ + h for h in self.heads if total in h.totals and h.variable not in dropped + ] + + def double_counts_in(self, total: str, active: Iterable[str] = ()) -> list: + dropped = self.dropped(total, active) + return [ + d + for d in self.double_counts + if total in d.totals and d.variable not in dropped + ] + + def components(self, total: str, active: Iterable[str] = ()) -> list: + """The variables the model should add for ``total``, per the partition.""" + return [h.variable for h in self.heads_in(total, active)] + [ + d.variable for d in self.double_counts_in(total, active) + ] + + def component_variables(self) -> list: + """Every variable named as a head or a declared double count, once.""" + return list( + dict.fromkeys( + [h.variable for h in self.heads] + + [d.variable for d in self.double_counts] + ) + ) + + def head_by_variable(self, variable: str) -> Optional[Head]: + for head in self.heads: + if head.variable == variable: + return head + return None + + def is_tax_total(self, total: str) -> bool: + spec = self.totals.get(total) + return spec is not None and spec.kind == "tax" + + def dropped_when_parameters(self) -> list: + return list( + dict.fromkeys( + p for spec in self.totals.values() for p, _ in spec.dropped_when + ) + ) + + def as_dict(self) -> dict: + return { + "source": self.source, + "sha256": self.sha256, + "schema_version": self.schema_version, + "tax_heads": [_head_dict(h) for h in self.tax_heads], + "spending_heads": [_head_dict(h) for h in self.spending_heads], + "model_double_counts": [ + { + "variable": d.variable, + "contained_in": d.contained_in, + "totals": list(d.totals), + "issue": d.issue, + } + for d in self.double_counts + ], + "totals": { + name: { + "kind": t.kind, + "module": t.module, + "list": t.list_name, + "dropped_when": {p: list(v) for p, v in t.dropped_when}, + } + for name, t in self.totals.items() + }, + } + + +def _head_dict(head: Head) -> dict: + return { + "id": head.id, + "label": head.label, + "variable": head.variable, + "totals": list(head.totals), + "report_group": head.report_group, + "kind": head.kind, + "entity": head.entity, + "obr_line": head.obr_line, + "payer": head.payer, + "sign": head.sign, + "zero_at_baseline": head.zero_at_baseline, + "caseload": head.caseload, + "notes": list(head.notes), + } + + +def group_label(report_group: str) -> str: + """``excise_duties`` -> ``Excise duties``; labels already in prose pass through.""" + text = report_group.replace("_", " ").strip() + return text[:1].upper() + text[1:] if text else "Other" + + +def default_partition_path() -> Optional[Path]: + """The partition shipped with the installed ``policyengine_uk``, if any.""" + spec = importlib.util.find_spec("policyengine_uk") + if spec is None or not spec.submodule_search_locations: + return None + for location in spec.submodule_search_locations: + candidate = Path(location) / PARTITION_RELATIVE_PATH + if candidate.exists(): + return candidate + return None + + +def _as_tuple(value, field_name: str, where: str) -> tuple: + if value is None: + return () + if isinstance(value, str): + raise PartitionError(f"{where}: {field_name} must be a list, not a string") + return tuple(str(v) for v in value) + + +def _parse_total(name: str, raw: Mapping) -> TotalSpec: + if raw is None: + raw = {} + if not isinstance(raw, Mapping): + raise PartitionError(f"totals.{name} must be a mapping") + known = KNOWN_TOTALS.get(name) + kind = raw.get("kind") or (known[0] if known else None) + module = raw.get("module") or (known[1] if known else None) + list_name = raw.get("list") or (known[2] if known else None) + if kind not in ("tax", "spending") or not module or not list_name: + raise PartitionError( + f"totals.{name}: an unfamiliar total needs kind (tax or spending), module and list" + ) + dropped = [] + for parameter, variables in (raw.get("dropped_when") or {}).items(): + dropped.append( + (str(parameter), _as_tuple(variables, "dropped_when", f"totals.{name}")) + ) + return TotalSpec( + name=name, + kind=kind, + module=str(module), + list_name=str(list_name), + label=str(raw.get("label") or ""), + dropped_when=tuple(dropped), + ) + + +def _parse_head(raw: Mapping, kind: str, index: int) -> Head: + where = f"{kind}_heads[{index}]" + if not isinstance(raw, Mapping): + raise PartitionError(f"{where}: must be a mapping") + missing = [k for k in ("id", "label", "variable", "totals") if not raw.get(k)] + if missing: + raise PartitionError(f"{where}: missing {missing}") + zero = raw.get("zero_at_baseline") + if zero not in (None, "expected"): + raise PartitionError(f"{where}: zero_at_baseline must be 'expected' or absent") + notes = tuple( + f"{key.replace('_', ' ')}: {' '.join(str(raw[key]).split())}" + for key in ("obr_line_note", "zero_at_baseline_note", "data_note", "model_note") + if raw.get(key) + ) + reads = tuple( + str(r.get("head")) for r in (raw.get("reads") or []) if isinstance(r, Mapping) + ) + caseload = raw.get("caseload") + return Head( + id=str(raw["id"]), + label=str(raw["label"]), + variable=str(raw["variable"]), + totals=_as_tuple(raw.get("totals"), "totals", where), + report_group=str(raw.get("report_group") or "other"), + kind=kind, + entity=raw.get("entity"), + obr_line=raw.get("obr_line"), + payer=raw.get("payer"), + sign=raw.get("sign"), + zero_at_baseline=zero == "expected", + caseload=bool(caseload) if caseload is not None else kind == "spending", + reads=reads, + notes=notes, + ) + + +def parse_partition( + raw: Mapping, source: str = "", sha256: str = "" +) -> Partition: + """Build a partition from parsed YAML, raising on any inconsistency.""" + if not isinstance(raw, Mapping): + raise PartitionError(f"{source}: the partition must be a mapping") + version = raw.get("schema_version") + if version not in SUPPORTED_SCHEMA_VERSIONS: + raise PartitionError( + f"{source}: schema_version {version!r} is not one of " + f"{sorted(SUPPORTED_SCHEMA_VERSIONS)}" + ) + totals = { + str(name): _parse_total(str(name), spec) + for name, spec in (raw.get("totals") or {}).items() + } + tax_heads = tuple( + _parse_head(h, "tax", i) for i, h in enumerate(raw.get("tax_heads") or []) + ) + spending_heads = tuple( + _parse_head(h, "spending", i) + for i, h in enumerate(raw.get("spending_heads") or []) + ) + double_counts = [] + for i, dc in enumerate(raw.get("model_double_counts") or []): + where = f"model_double_counts[{i}]" + if ( + not isinstance(dc, Mapping) + or not dc.get("variable") + or not dc.get("contained_in") + ): + raise PartitionError(f"{source}: {where} needs variable and contained_in") + double_counts.append( + DoubleCount( + variable=str(dc["variable"]), + contained_in=str(dc["contained_in"]), + totals=_as_tuple(dc.get("totals"), "totals", where), + issue=str(dc.get("issue") or ""), + ) + ) + exclusions = tuple( + Exclusion( + variable=str(e.get("variable")), + kind=str(e.get("kind") or ""), + head=str(e.get("head") or ""), + spans=_as_tuple(e.get("spans"), "spans", "exclusions"), + note=str(e.get("note") or e.get("reason") or ""), + ) + for e in (raw.get("exclusions") or []) + if isinstance(e, Mapping) + ) + partition = Partition( + schema_version=version, + totals=totals, + tax_heads=tax_heads, + spending_heads=spending_heads, + double_counts=tuple(double_counts), + exclusions=exclusions, + overlap_guards=tuple(raw.get("overlap_guards") or ()), + source=source, + sha256=sha256, + raw=raw, + ) + problems = structural_problems(partition) + if problems: + raise PartitionError(f"{source}:\n- " + "\n- ".join(problems)) + return partition + + +def load_partition(path) -> Partition: + path = Path(path) + data = path.read_bytes() + return parse_partition( + yaml.safe_load(data), source=str(path), sha256=hashlib.sha256(data).hexdigest() + ) + + +def structural_problems(partition: Partition) -> list: + """Inconsistencies visible from the partition alone, without the model.""" + problems = [] + heads = partition.heads + head_ids = {h.id for h in heads} + for name, count in Counter(h.id for h in heads).items(): + if count > 1: + problems.append(f"head id {name!r} is used {count} times") + for name, count in Counter(h.variable for h in heads).items(): + if count > 1: + problems.append(f"variable {name!r} is the variable of {count} heads") + head_variables = {h.variable for h in heads} + for head in heads: + if len(set(head.totals)) != len(head.totals): + problems.append(f"head {head.id!r} lists a total twice") + for total in head.totals: + spec = partition.totals.get(total) + if spec is None: + problems.append(f"head {head.id!r} names unknown total {total!r}") + elif spec.kind != head.kind: + problems.append( + f"{head.kind} head {head.id!r} is in {total!r}, a {spec.kind} total" + ) + if head.entity is not None and head.entity not in ENTITIES: + problems.append(f"head {head.id!r} has unknown entity {head.entity!r}") + if head.sign not in SIGNS: + problems.append(f"head {head.id!r} has unknown sign {head.sign!r}") + for read in head.reads: + if read not in head_ids: + problems.append( + f"head {head.id!r} declares a read of unknown head {read!r}" + ) + for name, spec in partition.totals.items(): + listed = set(partition.components(name)) + for parameter, variables in spec.dropped_when: + for variable in variables: + if variable not in listed: + problems.append( + f"totals.{name} drops {variable!r} under {parameter}, but no head " + "or double count of that total is that variable" + ) + for dc in partition.double_counts: + if dc.variable in head_variables: + problems.append( + f"double count {dc.variable!r} is also a head; it would be counted twice" + ) + container = partition.head_by_variable(dc.contained_in) + if container is None: + problems.append( + f"double count {dc.variable!r} is contained in {dc.contained_in!r}, " + "which is not a head variable" + ) + else: + missing = [t for t in dc.totals if t not in container.totals] + if missing: + problems.append( + f"double count {dc.variable!r} is in {missing}, which its " + f"container {dc.contained_in!r} is not" + ) + for total in dc.totals: + if total not in partition.totals: + problems.append( + f"double count {dc.variable!r} names unknown total {total!r}" + ) + for name, count in Counter(d.variable for d in partition.double_counts).items(): + if count > 1: + problems.append(f"double count {name!r} is declared {count} times") + for exclusion in partition.exclusions: + if exclusion.variable in head_variables: + problems.append( + f"excluded variable {exclusion.variable!r} is also a head variable" + ) + for referenced in ([exclusion.head] if exclusion.head else []) + list( + exclusion.spans + ): + if referenced not in head_ids: + problems.append( + f"exclusion {exclusion.variable!r} refers to unknown head {referenced!r}" + ) + return problems + + +def read_model_lists(partition: Partition) -> tuple: + """Import each total's component list from the model. + + Returns ``(lists, problems)``: the lists that could be read, keyed by + total, and a description of every total whose list could not be read. + """ + lists, problems = {}, [] + for name, spec in partition.totals.items(): + try: + module = importlib.import_module(spec.module) + values = getattr(module, spec.list_name) + except (ImportError, AttributeError) as error: + problems.append( + f"{name}: cannot read {spec.module}.{spec.list_name}: {error}" + ) + continue + lists[name] = [str(v) for v in values] + return lists, problems + + +def closure_problems( + partition: Partition, + model_lists: Mapping[str, Iterable[str]], + model_variables: Optional[Mapping] = None, +) -> list: + """Differences between the partition and the model's own component lists. + + For every total, the heads and double counts naming it must be exactly + the model's list, each listed once; every named variable must exist, and + a head's declared entity must be its variable's entity. + """ + problems = [] + for total, listed in model_lists.items(): + listed = list(listed) + for variable, count in Counter(listed).items(): + if count > 1: + problems.append(f"{total}: the model lists {variable!r} {count} times") + expected, assigned = set(listed), set(partition.components(total)) + for variable in sorted(expected - assigned): + problems.append( + f"{total}: the model adds {variable!r}, which no head or declared " + "double count covers" + ) + for variable in sorted(assigned - expected): + problems.append( + f"{total}: the partition puts {variable!r} in this total, but the " + "model does not add it" + ) + if model_variables is not None: + named = partition.component_variables() + [ + d.contained_in for d in partition.double_counts + ] + for variable in sorted(set(named) - set(model_variables)): + problems.append(f"{variable!r} is not a model variable") + for head in partition.heads: + definition = ( + model_variables.get(head.variable) + if hasattr(model_variables, "get") + else None + ) + if ( + head.entity + and definition is not None + and definition.entity.key != head.entity + ): + problems.append( + f"head {head.id!r} says entity {head.entity!r}, but {head.variable} " + f"is a {definition.entity.key} variable" + ) + return problems diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py new file mode 100644 index 0000000000..c57c4ab13d --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -0,0 +1,375 @@ +"""Render a comparison as a Markdown pull request comment.""" + +from __future__ import annotations + +import math +from typing import Mapping, Optional + +from . import COMMENT_MARKER +from .spec import Spec, fiscal_year_label + +SECTIONS = ( + ("receipts", "Receipts by tax head"), + ("spending", "Spending by benefit and service"), + ("caseloads", "Caseloads"), + ("balance", "Government balance, each component counted once"), + ("model_totals", "Model totals"), + ("double_counts", "Components the model counts twice"), + ("household_totals", "Household income totals"), + ("population", "Population"), + ("poverty", "Poverty rates"), + ("poverty_count", "People in poverty"), + ("inequality", "Inequality"), + ("deciles", "Mean household net income in each side's own deciles"), + ("reforms", "Standard reform costings (change in gov_balance)"), +) +MAX_FLAG_ROWS = 40 + + +def _finite(value) -> bool: + return isinstance(value, (int, float)) and math.isfinite(value) + + +def fmt_level(value, kind) -> str: + if not _finite(value): + return "n/a" + if kind == "gbp": + return f"£{value / 1e9:,.2f}bn" + if kind == "gbp_household": + return f"£{value:,.0f}" + if kind == "count": + return f"{value / 1e3:,.0f}k" + if kind == "rate": + return f"{100 * value:.1f}%" + return f"{value:.4f}" + + +def fmt_change(value, kind) -> str: + if value is None or not _finite(value): + return "n/a" + if value == 0: + return "0" + sign = "+" if value > 0 else "-" + size = abs(value) + if kind == "gbp": + if size >= 1e9: + return f"{sign}£{size / 1e9:,.2f}bn" + if size >= 1e5: + return f"{sign}£{size / 1e6:,.1f}m" + return f"{sign}£{size:,.0f}" + if kind == "gbp_household": + return f"{sign}£{size:,.2f}" if size < 10 else f"{sign}£{size:,.0f}" + if kind == "count": + if size >= 1e3: + return f"{sign}{size / 1e3:,.1f}k" + return f"{sign}{size:,.0f}" + if kind == "rate": + return f"{sign}{100 * size:.2f}pp" + return f"{sign}{size:.4f}" + + +def _table(header, rows) -> str: + lines = [ + "| " + " | ".join(header) + " |", + "|" + "|".join(["---"] + ["---:"] * (len(header) - 1)) + "|", + ] + lines += ["| " + " | ".join(str(c) for c in row) + " |" for row in rows] + return "\n".join(lines) + + +def _year_header(years): + return [fiscal_year_label(int(y)) for y in years] + + +def _section_rows(summary, section): + return [r for r in summary["rows"] if r["section"] == section] + + +def _changes_table(rows, years) -> str: + changed = [r for r in rows if r["changed"]] + if not changed: + return "No change in any year." + body = [] + for row in changed: + label = row["label"] if not row.get("group") else f"{row['label']}" + body.append( + [label] + + [fmt_change(row["values"][str(y)]["delta"], row["kind"]) for y in years] + ) + return _table(["Measure"] + _year_header(years), body) + + +def _levels_table(rows, years) -> str: + body = [ + [row["label"]] + + [fmt_level(row["values"][str(y)]["head"], row["kind"]) for y in years] + for row in rows + ] + return _table(["Measure (head)"] + _year_header(years), body) + + +def _details(summary_text, body) -> str: + return f"
\n{summary_text}\n\n{body}\n\n
" + + +def _deciles(summary, years) -> str: + deciles = summary.get("deciles") or {} + if not deciles: + return "Not available: both sides need household detail." + present = [y for y in years if str(y) in deciles] + labels = [row["decile"] for row in deciles[str(present[0])]] + body = [] + for index, decile in enumerate(labels): + cells = [] + for year in present: + row = deciles[str(year)][index] + if row["mean_change"] == 0 or not _finite(row["mean_change"]): + cells.append("0" if row["mean_change"] == 0 else "n/a") + else: + cells.append( + f"{fmt_change(row['mean_change'], 'gbp')} ({100 * row['relative_change']:+.2f}%)" + ) + body.append([decile if decile == "all" else f"Decile {decile}"] + cells) + text = ( + "Mean change in household net income per household, and as a share of the " + "decile's base income; households stay in the base run's deciles.\n\n" + + _table(["Base decile"] + _year_header(present), body) + ) + winners = [] + for year in present: + overall = deciles[str(year)][-1] + winners.append( + [ + fiscal_year_label(year), + f"{100 * overall['share_gaining']:.1f}%", + f"{100 * overall['share_losing']:.1f}%", + ] + ) + text += "\n\n" + _table( + ["Year", "People gaining over £1", "People losing over £1"], winners + ) + if summary.get("decile_notes"): + text += "\n\n" + "\n".join(f"- {note}" for note in summary["decile_notes"]) + return text + + +def _flags(summary) -> str: + flags = list(summary.get("flags") or []) + flags += [ + { + "label": f"Decile {f['decile']} household net income, relative", + "year": f["year"], + "kind": "relative", + "base": None, + "head": None, + "delta": f["relative_change"], + "limit": None, + "section": "deciles", + } + for f in summary.get("decile_flags") or [] + ] + if not flags: + return "None." + flags.sort(key=lambda f: -(abs(f["delta"]) / f["limit"]) if f.get("limit") else 0) + shown = flags[:MAX_FLAG_ROWS] + body = [] + for f in shown: + if f["kind"] == "relative": + body.append( + [ + f["label"], + fiscal_year_label(f["year"]), + "", + "", + f"{100 * f['delta']:+.2f}%", + ] + ) + else: + body.append( + [ + f["label"], + fiscal_year_label(f["year"]), + fmt_level(f["base"], f["kind"]), + fmt_level(f["head"], f["kind"]), + fmt_change(f["delta"], f["kind"]), + ] + ) + text = _table(["Measure", "Year", "Base", "Head", "Change"], body) + if len(flags) > len(shown): + text += f"\n\n{len(flags) - len(shown)} more flags are in the report artifact." + return text + + +def _check_list(checks, classification) -> list: + out = [] + for check in checks: + if check["classification"] != classification: + continue + details = check["head_details"] or check["base_details"] + detail = f": {'; '.join(details[:3])}" if details else "" + out.append(f"- {check['label']} (`{check['id']}`){detail}") + return out + + +def _guard(summary) -> str: + guard = (summary.get("head") or {}).get("guard") or {} + if guard.get("status") != "ran": + return f"Counted-once check not run: {guard.get('reason', 'unknown reason')}." + components = guard.get("components", []) + tested = [c for c in components if c["status"] != "skipped"] + failed = [c for c in tested if c["status"] == "fail"] + lines = [ + f"{len(tested)} components perturbed in {fiscal_year_label(guard['year'])}; " + f"{len(failed)} counted other than once in the partition; " + f"{len(components) - len(tested)} skipped (zero for every unit, or not money)." + ] + model = [ + (c["variable"], total, k) + for c in tested + for total, k in c.get("model_coefficients", {}).items() + if abs(k - 1) > guard.get("tolerance", 1e-3) + ] + if model: + lines.append("") + lines.append("Components the model's own totals count other than once:") + lines += [f"- `{v}` moves `{t}` by {k:.3f} per unit" for v, t, k in model] + for dc in guard.get("double_counts", []): + state = "still contained" if dc["status"] == "pass" else "NO LONGER contained" + lines.append( + f"- declared double count `{dc['variable']}` is {state} in `{dc['contained_in']}` " + f"(coefficient {dc['coefficient']:.3f}; {dc['issue']})" + ) + return "\n".join(lines) + + +def _provenance(summary) -> str: + rows = [] + for side in ("base", "head"): + info = summary.get(side) + if not info: + rows.append([side, "no result", "", "", "", ""]) + continue + provenance = info.get("provenance") or {} + runtime = info.get("runtime") or {} + rows.append( + [ + side, + f"{provenance.get('git_ref', '')} `{str(provenance.get('git_sha', ''))[:10]}`", + f"uk {runtime.get('policyengine_uk')}, core {runtime.get('policyengine_core')}, py {runtime.get('python')} {runtime.get('machine', '')}", + f"`{str(runtime.get('distributions_sha256', ''))[:12]}`", + f"`{str((info.get('partition') or {}).get('sha256', ''))[:12]}`", + f"{(info.get('timings') or {}).get('total', '')}s", + ] + ) + dataset = ((summary.get("head") or {}).get("provenance") or {}).get("dataset") or {} + text = _table( + ["Side", "Commit", "Runtime", "Environment hash", "Partition hash", "Time"], + rows, + ) + if dataset: + text += ( + f"\n\nDataset: `{dataset.get('uri')}`, sha256 `{dataset.get('verified_sha256')}` " + f"(verified on both sides), {dataset.get('data_package')} {dataset.get('data_version')}." + ) + if summary.get("runtime_differences"): + text += "\n\nRuntime differences:\n" + "\n".join( + f"- {d}" for d in summary["runtime_differences"] + ) + return text + + +def render(summary: Mapping, spec: Spec, context: Optional[Mapping] = None) -> str: + """The full comment; sections are dropped from the end to fit GitHub's limit.""" + context = context or {} + years = summary.get("years") or spec.years + regressions = [c for c in summary["checks"] if c["classification"] == "regression"] + pre_existing = [ + c for c in summary["checks"] if c["classification"] == "pre_existing" + ] + new_warnings = [ + c for c in summary["checks"] if c["classification"] == "new_warning" + ] + fixed = [c for c in summary["checks"] if c["classification"] == "fixed"] + flag_count = len(summary.get("flags") or []) + len( + summary.get("decile_flags") or [] + ) + verdict = "FAIL" if summary["status"] == "fail" else "pass" + + head_prov = (summary.get("head") or {}).get("provenance") or {} + base_prov = (summary.get("base") or {}).get("provenance") or {} + dataset = head_prov.get("dataset") or {} + parts = [ + COMMENT_MARKER, + f"## Aggregate impact on certified data: {verdict}", + "", + ( + f"Base `{str(base_prov.get('git_sha', '?'))[:10]}` against head " + f"`{str(head_prov.get('git_sha', '?'))[:10]}`, both on `{dataset.get('path', '?')}` " + f"({dataset.get('data_package', '')} {dataset.get('data_version', '')}, sha256 " + f"`{str(dataset.get('verified_sha256', '?'))[:12]}`), fiscal years " + f"{fiscal_year_label(years[0])} to {fiscal_year_label(years[-1])}. Each year is a full " + "microsimulation run on each side." + ), + "", + ( + f"**Checks:** {len(regressions)} broken by this change, {len(pre_existing)} failing on " + f"both sides, {len(fixed)} fixed. **Moves flagged for review:** {flag_count} " + "(thresholds proposed in `report_spec.yaml`, pending cos d1344; a flag does not fail the job)." + ), + ] + if context.get("run_url"): + parts.append(f"\n[Workflow run and full report]({context['run_url']})") + if summary["failures"]: + parts += ["", "### Failures", ""] + [f"- {f}" for f in summary["failures"]] + if new_warnings: + parts += ["", "### New warnings", ""] + _check_list( + summary["checks"], "new_warning" + ) + if summary["warnings"]: + parts += ["", "### Notes", ""] + [f"- {w}" for w in summary["warnings"]] + if fixed: + parts += ["", "### Fixed by this change", ""] + _check_list( + summary["checks"], "fixed" + ) + parts += ["", "### Moves flagged for review", "", _flags(summary)] + + for section, title in SECTIONS: + rows = _section_rows(summary, section) + if not rows: + continue + parts += ["", f"### {title}", "", _changes_table(rows, years)] + parts += ["", "### Household net income by decile", "", _deciles(summary, years)] + + optional = [] + if pre_existing: + optional.append( + _details( + f"Checks failing on both base and head ({len(pre_existing)})", + "\n".join(_check_list(summary["checks"], "pre_existing")), + ) + ) + optional.append(_details("Counted-once check", _guard(summary))) + optional.append(_details("Provenance", _provenance(summary))) + for section, title in SECTIONS: + rows = _section_rows(summary, section) + if rows: + optional.append( + _details(f"{title}: head levels", _levels_table(rows, years)) + ) + + text = "\n".join(parts) + limit = spec.max_comment_characters + note = "\n\n_Some detail was left out to fit a GitHub comment; the workflow artifact has all of it._" + kept = [] + for block in optional: + candidate = text + "\n\n" + "\n\n".join(kept + [block]) + if len(candidate) + len(note) > limit: + continue + kept.append(block) + if kept: + text += "\n\n" + "\n\n".join(kept) + if len(kept) < len(optional): + text += note + if len(text) > limit: + text = text[: limit - len(note) - 1] + note + return text + "\n" diff --git a/policyengine_uk/utils/aggregate_impact/report_spec.yaml b/policyengine_uk/utils/aggregate_impact/report_spec.yaml new file mode 100644 index 0000000000..04fb81640a --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/report_spec.yaml @@ -0,0 +1,88 @@ +# What the aggregate-impact report measures and when it flags a move. +schema_version: 1 + +# Model periods are fiscal years: 2026 is 2026-27. The horizon spans the +# current fiscal year's predecessor (2025-26, the last year of outturn the +# Budget compares against) through 2031-32, the last year of the Autumn +# Budget 2026 scorecard. Every year is a full microsimulation run. +horizon: + first_year: 2025 + last_year: 2031 + +# Year of the counted-once check, which perturbs every component once. +guard_year: 2026 + +poverty: + measures: + - id: absolute_bhc + label: Absolute poverty, before housing costs + variable: in_poverty_bhc + - id: absolute_ahc + label: Absolute poverty, after housing costs + variable: in_poverty_ahc + - id: relative_bhc + label: Relative poverty, before housing costs + variable: in_relative_poverty_bhc + - id: relative_ahc + label: Relative poverty, after housing costs + variable: in_relative_poverty_ahc + groups: + - id: all + label: Everyone + - id: children + label: Children + variable: is_hbai_dependent_child + - id: working_age_adults + label: Working-age adults + variable: is_hbai_working_age_adult + - id: pensioners + label: Pensioners + variable: is_hbai_pensioner + +inequality: + - id: gini_equiv_hbai_bhc + label: Gini coefficient, equivalised HBAI income before housing costs + variable: equiv_hbai_household_net_income + +deciles: + # Households are placed in deciles by the base run's own decile variable + # (person-weighted equivalised HBAI income); head incomes are compared + # household by household within those base deciles. + decile_variable: household_income_decile + income_variable: household_net_income + +# A move is flagged for review when |head - base| exceeds +# max(absolute, relative * |base|). Flags never fail the job: intended policy +# changes move numbers. These thresholds are proposals awaiting the +# numerical-acceptance ruling (cos d1344). +flags: + gbp: + absolute: 100000000 + relative: 0.001 + count: + absolute: 5000 + relative: 0.005 + rate: + absolute: 0.001 + index: + absolute: 0.001 + decile_relative: + absolute: 0.001 + gbp_household: + absolute: 10 + relative: 0.001 + reform_gbp: + absolute: 100000000 + relative: 0.01 + +# Relative tolerance for accounting identities between weighted totals. The +# model stores values as float32, so totals summed in different orders differ +# by a few parts in ten million. +identity_relative_tolerance: 0.000001 +identity_absolute_tolerance_gbp: 1000000 + +# Tolerance on the counted-once coefficient (1 per unit perturbed). +counted_once_tolerance: 0.001 + +comment: + max_characters: 60000 diff --git a/policyengine_uk/utils/aggregate_impact/spec.py b/policyengine_uk/utils/aggregate_impact/spec.py new file mode 100644 index 0000000000..2441253702 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/spec.py @@ -0,0 +1,105 @@ +"""The report specification: horizon, measures and flag thresholds.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Optional + +import yaml + +DEFAULT_SPEC_PATH = Path(__file__).with_name("report_spec.yaml") + + +class SpecError(ValueError): + pass + + +@dataclass(frozen=True) +class Threshold: + absolute: float + relative: float = 0.0 + + def limit(self, base: float) -> float: + return max(self.absolute, self.relative * abs(base)) + + def exceeded(self, base: float, head: float) -> bool: + return abs(head - base) > self.limit(base) + + +@dataclass(frozen=True) +class Spec: + raw: Mapping + first_year: int + last_year: int + guard_year: int + poverty_measures: tuple + poverty_groups: tuple + inequality: tuple + decile_variable: str + income_variable: str + flags: Mapping[str, Threshold] + identity_relative_tolerance: float + identity_absolute_tolerance: float + counted_once_tolerance: float + max_comment_characters: int + + @property + def years(self) -> list: + return list(range(self.first_year, self.last_year + 1)) + + def threshold(self, kind: str) -> Optional[Threshold]: + return self.flags.get(kind) + + +def fiscal_year_label(year: int) -> str: + return f"{year}-{(year + 1) % 100:02d}" + + +def parse_spec(raw: Mapping) -> Spec: + if raw.get("schema_version") != 1: + raise SpecError( + f"unsupported spec schema_version {raw.get('schema_version')!r}" + ) + horizon = raw.get("horizon") or {} + first, last = int(horizon["first_year"]), int(horizon["last_year"]) + if last < first: + raise SpecError("horizon.last_year is before horizon.first_year") + guard_year = int(raw.get("guard_year", first)) + poverty = raw.get("poverty") or {} + flags = { + kind: Threshold(float(v.get("absolute", 0.0)), float(v.get("relative", 0.0))) + for kind, v in (raw.get("flags") or {}).items() + } + deciles = raw.get("deciles") or {} + return Spec( + raw=raw, + first_year=first, + last_year=last, + guard_year=guard_year, + poverty_measures=tuple(poverty.get("measures") or ()), + poverty_groups=tuple(poverty.get("groups") or ()), + inequality=tuple(raw.get("inequality") or ()), + decile_variable=str(deciles.get("decile_variable", "household_income_decile")), + income_variable=str(deciles.get("income_variable", "household_net_income")), + flags=flags, + identity_relative_tolerance=float(raw.get("identity_relative_tolerance", 1e-6)), + identity_absolute_tolerance=float( + raw.get("identity_absolute_tolerance_gbp", 1e6) + ), + counted_once_tolerance=float(raw.get("counted_once_tolerance", 1e-3)), + max_comment_characters=int( + (raw.get("comment") or {}).get("max_characters", 60000) + ), + ) + + +def load_spec(path=None, years: Optional[str] = None) -> Spec: + """Load the spec, optionally overriding the horizon with ``"2025-2031"``.""" + with open(path or DEFAULT_SPEC_PATH) as f: + raw = yaml.safe_load(f) + if years: + start, _, end = years.partition("-") + raw = dict(raw) + raw["horizon"] = {"first_year": int(start), "last_year": int(end or start)} + return parse_spec(raw) From 073e80809141852eda42127b9e692b17e81343eb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 13:45:41 -0400 Subject: [PATCH 02/10] Run the aggregate-impact report on every pull request .github/workflows/aggregate-impact.yaml measures the pull request's base and head on the same runner image. Each side syncs its own uv.lock and runs the head's copy of the tool against its own model, on the pinned dataset whose bytes the tool verifies. The report job then compares the two, writes the job summary and one updated comment, and fails on structural failures. Fork and Dependabot runs, which cannot read the private dataset, record a skip instead. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/aggregate-impact.yaml | 238 ++++++++++++++++++ .../test_aggregate_impact_compare.py | 8 + .../utils/aggregate_impact/compare.py | 17 ++ .../utils/aggregate_impact/render.py | 3 + 4 files changed, 266 insertions(+) create mode 100644 .github/workflows/aggregate-impact.yaml diff --git a/.github/workflows/aggregate-impact.yaml b/.github/workflows/aggregate-impact.yaml new file mode 100644 index 0000000000..9b0f14e2bc --- /dev/null +++ b/.github/workflows/aggregate-impact.yaml @@ -0,0 +1,238 @@ +# Measure a pull request's base and head on the certified dataset and post +# the comparison: receipts by tax head, spending and caseloads, poverty, +# inequality, deciles and the standard reform costings, 2025-26 to 2031-32. +# +# Both sides run the pull request head's copy of the measurement tool +# (policyengine_uk/utils/aggregate_impact), each in an environment synced +# from its own uv.lock, on the dataset pinned in +# policyengine_uk/tests/certified_dataset.yaml (bytes verified on each side). +# Each side reads its own fiscal partition. The report job fails on a crash, +# a different dataset, non-finite figures or a check the head breaks that the +# base passes; moves above the flag thresholds are listed, not failed. + +name: Aggregate impact + +on: + pull_request: + branches: + - main + paths: + - policyengine_uk/** + - pyproject.toml + - uv.lock + - .github/workflows/aggregate-impact.yaml + workflow_dispatch: + inputs: + base: + description: Base commit, branch or tag + default: main + head: + description: Head commit, branch or tag (default the dispatched ref) + default: "" + +concurrency: + group: aggregate-impact-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + UV_PYTHON: "3.13" + PYTHONUNBUFFERED: "1" + +jobs: + measure: + name: Measure ${{ matrix.side }} + # Fork pull requests cannot read the private certified dataset. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + # The same runner image for both sides, as for the Test job. + runs-on: ubuntu-24.04-arm + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + side: [base, head] + steps: + - name: Choose the commits + id: commits + env: + EVENT: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + INPUT_BASE: ${{ inputs.base }} + INPUT_HEAD: ${{ inputs.head }} + run: | + # A pull request's head is its merge commit, whose first parent is + # the base, so the comparison is exactly what merging would change. + if [ "$EVENT" = "pull_request" ]; then + base="$PR_BASE"; head="$GITHUB_SHA" + else + base="${INPUT_BASE:-main}"; head="${INPUT_HEAD:-$GITHUB_SHA}" + fi + echo "base=$base" >> "$GITHUB_OUTPUT" + echo "head=$head" >> "$GITHUB_OUTPUT" + if [ "${{ matrix.side }}" = "base" ]; then + echo "model=$base" >> "$GITHUB_OUTPUT" + else + echo "model=$head" >> "$GITHUB_OUTPUT" + fi + + - name: Check out the head (tool, pin, partition fallback) + uses: actions/checkout@v6 + with: + ref: ${{ steps.commits.outputs.head }} + path: tool-source + + - name: Check out the ${{ matrix.side }} model + uses: actions/checkout@v6 + with: + ref: ${{ steps.commits.outputs.model }} + path: model + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Sync the ${{ matrix.side }} environment from its lock + working-directory: model + run: uv sync --frozen --no-dev + + - name: Restore the certified dataset + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface/hub/models--policyengine--policyengine-uk-data-private + key: certified-data-${{ hashFiles('tool-source/policyengine_uk/tests/certified_dataset.yaml') }} + + - name: Measure + working-directory: model + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ACTOR: ${{ github.actor }} + SIDE: ${{ matrix.side }} + MODEL_SHA: ${{ steps.commits.outputs.model }} + run: | + if [ -z "$HUGGING_FACE_TOKEN" ]; then + if [ "$ACTOR" = "dependabot[bot]" ]; then + echo "::notice::Dependabot runs have no data access; the aggregate impact report is skipped." + mkdir -p "$GITHUB_WORKSPACE/out/$SIDE" + echo '{"schema": "policyengine-uk-aggregate-impact/1", "status": "skipped", "reason": "Dependabot runs have no access to the certified dataset"}' \ + > "$GITHUB_WORKSPACE/out/$SIDE/result.json" + exit 0 + fi + echo "::error::HUGGING_FACE_TOKEN is not available, so the certified dataset cannot be read." + exit 1 + fi + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" + cp -R ../tool-source/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + set +e + PYTHONPATH="$tool" .venv/bin/python -m pe_uk_aggregate_impact measure \ + --label "$SIDE" \ + --pin ../tool-source/policyengine_uk/tests/certified_dataset.yaml \ + --reforms ../tool-source/policyengine_uk/tests/microsimulation/reforms_config.yaml \ + --fallback-partition ../tool-source/policyengine_uk/data/fiscal_partition.yaml \ + --git-sha "$(git rev-parse HEAD)" \ + --git-ref "$MODEL_SHA" \ + --out "$GITHUB_WORKSPACE/out/$SIDE" + status=$? + set -e + # A crash is recorded in result.json for the report to judge; only + # a run that wrote nothing fails here. + if [ ! -f "$GITHUB_WORKSPACE/out/$SIDE/result.json" ]; then + echo "::error::The measurement wrote no result (exit $status)." + exit 1 + fi + + - name: Upload the ${{ matrix.side }} measurement + if: always() + uses: actions/upload-artifact@v4 + with: + name: aggregate-impact-${{ matrix.side }} + path: out/${{ matrix.side }} + if-no-files-found: ignore + retention-days: 30 + + report: + name: Report + needs: measure + if: >- + always() && ( + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + ) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Check out the head + uses: actions/checkout@v6 + with: + ref: ${{ github.event_name == 'pull_request' && github.sha || inputs.head || github.sha }} + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Download the measurements + uses: actions/download-artifact@v4 + with: + pattern: aggregate-impact-* + path: out + + - name: Compare base with head + id: compare + run: | + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" + cp -R policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + uv venv -q "$RUNNER_TEMP/compare-env" + uv pip install -q --python "$RUNNER_TEMP/compare-env" numpy pyyaml + base_args=() + if [ -f out/aggregate-impact-base/result.json ]; then + base_args=(--base out/aggregate-impact-base) + fi + mkdir -p out/aggregate-impact-head report + set +e + PYTHONPATH="$tool" "$RUNNER_TEMP/compare-env/bin/python" -m pe_uk_aggregate_impact compare \ + "${base_args[@]}" \ + --head out/aggregate-impact-head \ + --markdown report/comment.md \ + --summary report/summary.json \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + echo "status=$?" >> "$GITHUB_OUTPUT" + set -e + cat report/comment.md >> "$GITHUB_STEP_SUMMARY" + + - name: Post or update the pull request comment + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + marker='' + existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \ + | tail -n 1) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@report/comment.md > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@report/comment.md > /dev/null + fi + + - name: Upload the report + if: always() + uses: actions/upload-artifact@v4 + with: + name: aggregate-impact-report + path: report + if-no-files-found: ignore + retention-days: 30 + + - name: Fail on a structural failure + if: steps.compare.outputs.status != '0' + run: | + echo "::error::The aggregate impact report found failures; see the job summary." + exit 1 diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py index d420b31662..1661ac49e0 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py @@ -309,3 +309,11 @@ def test_the_comment_carries_the_marker_and_fits_its_limit(values, limit): assert text.startswith(COMMENT_MARKER) assert len(text) <= limit + 1 assert render(summary, spec) == text + + +def test_a_skipped_head_passes_with_a_note(): + skipped = {"schema": TOOL_SCHEMA, "status": "skipped", "reason": "no data access"} + summary = compare(None, skipped, SPEC) + assert summary["status"] == "skipped" and summary["failures"] == [] + text = render(summary, SPEC) + assert text.startswith(COMMENT_MARKER) and "no data access" in text diff --git a/policyengine_uk/utils/aggregate_impact/compare.py b/policyengine_uk/utils/aggregate_impact/compare.py index d8ef4a5ef9..f529e457dd 100644 --- a/policyengine_uk/utils/aggregate_impact/compare.py +++ b/policyengine_uk/utils/aggregate_impact/compare.py @@ -234,6 +234,23 @@ def compare( strict: bool = False, ) -> dict: failures, warnings = [], [] + if head and head.get("status") == "skipped": + return { + "schema": TOOL_SCHEMA + "/comparison", + "status": "skipped", + "failures": [], + "warnings": [f"Not measured: {head.get('reason', 'no reason recorded')}"], + "years": spec.years, + "checks": [], + "rows": [], + "flags": [], + "deciles": {}, + "decile_notes": [], + "decile_flags": [], + "runtime_differences": [], + "base": _side_summary(base), + "head": _side_summary(head), + } head_ok = bool(head) and head.get("status") == "ok" base_ok = bool(base) and base.get("status") == "ok" if not head: diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py index c57c4ab13d..be6f916ab9 100644 --- a/policyengine_uk/utils/aggregate_impact/render.py +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -281,6 +281,9 @@ def _provenance(summary) -> str: def render(summary: Mapping, spec: Spec, context: Optional[Mapping] = None) -> str: """The full comment; sections are dropped from the end to fit GitHub's limit.""" context = context or {} + if summary["status"] == "skipped": + notes = "\n".join(f"- {w}" for w in summary.get("warnings") or []) + return f"{COMMENT_MARKER}\n## Aggregate impact on certified data: skipped\n\n{notes}\n" years = summary.get("years") or spec.years regressions = [c for c in summary["checks"] if c["classification"] == "regression"] pre_existing = [ From 99909e943659d6d5598ffef4cfd956a922d22763 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 14:36:50 -0400 Subject: [PATCH 03/10] Keep the aggregate-impact comment short when nothing changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sections with no change collapse into one line, an unchanged decile table into one sentence, and the counted-once summary lists only real violations (a tax total moving by other than £1 per unit, or a spending total by more than £1), not the means-test offsets every benefit shows. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .../utils/aggregate_impact/render.py | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py index be6f916ab9..374e4f7722 100644 --- a/policyengine_uk/utils/aggregate_impact/render.py +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -117,6 +117,8 @@ def _deciles(summary, years) -> str: if not deciles: return "Not available: both sides need household detail." present = [y for y in years if str(y) in deciles] + if all(row["mean_change"] == 0 for y in present for row in deciles[str(y)]): + return "No household's net income changes in any year." labels = [row["decile"] for row in deciles[str(present[0])]] body = [] for index, decile in enumerate(labels): @@ -215,29 +217,39 @@ def _guard(summary) -> str: guard = (summary.get("head") or {}).get("guard") or {} if guard.get("status") != "ran": return f"Counted-once check not run: {guard.get('reason', 'unknown reason')}." + tolerance = guard.get("tolerance", 1e-3) + tax_totals = {"gov_tax", "household_tax", "pre_budget_change_household_tax"} components = guard.get("components", []) tested = [c for c in components if c["status"] != "skipped"] failed = [c for c in tested if c["status"] == "fail"] lines = [ - f"{len(tested)} components perturbed in {fiscal_year_label(guard['year'])}; " - f"{len(failed)} counted other than once in the partition; " - f"{len(components) - len(tested)} skipped (zero for every unit, or not money)." + f"Each of {len(tested)} components was raised by £1 per unit in " + f"{fiscal_year_label(guard['year'])}: tax totals must rise by exactly £1 " + "per unit and spending totals by at most £1. " + f"{len(failed)} failed; {len(components) - len(tested)} were skipped " + "(zero for every unit, or not money)." ] + for c in failed: + moves = ", ".join(f"`{t}` {k:.4f}" for t, k in c["coefficients"].items()) + lines.append(f"- `{c['variable']}`: {moves}") + + def violates(total, k): + return abs(k - 1) > tolerance if total in tax_totals else k > 1 + tolerance + model = [ (c["variable"], total, k) for c in tested for total, k in c.get("model_coefficients", {}).items() - if abs(k - 1) > guard.get("tolerance", 1e-3) + if violates(total, k) ] if model: - lines.append("") - lines.append("Components the model's own totals count other than once:") - lines += [f"- `{v}` moves `{t}` by {k:.3f} per unit" for v, t, k in model] + lines += ["", "In the model's own totals:"] + lines += [f"- `{v}` moves `{t}` by {k:.4f} per unit" for v, t, k in model] for dc in guard.get("double_counts", []): state = "still contained" if dc["status"] == "pass" else "NO LONGER contained" lines.append( - f"- declared double count `{dc['variable']}` is {state} in `{dc['contained_in']}` " - f"(coefficient {dc['coefficient']:.3f}; {dc['issue']})" + f"- declared double count `{dc['variable']}` is {state} in " + f"`{dc['contained_in']}` (coefficient {dc['coefficient']:.3f}; {dc['issue']})" ) return "\n".join(lines) @@ -336,12 +348,18 @@ def render(summary: Mapping, spec: Spec, context: Optional[Mapping] = None) -> s ) parts += ["", "### Moves flagged for review", "", _flags(summary)] + unchanged = [] for section, title in SECTIONS: rows = _section_rows(summary, section) if not rows: continue + if not any(row["changed"] for row in rows): + unchanged.append(title[:1].lower() + title[1:]) + continue parts += ["", f"### {title}", "", _changes_table(rows, years)] parts += ["", "### Household net income by decile", "", _deciles(summary, years)] + if unchanged: + parts += ["", "**No change in any year:** " + "; ".join(unchanged) + "."] optional = [] if pre_existing: From cbd2fedcd111d4f9dd3471492ce155c22c487584 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 14:42:49 -0400 Subject: [PATCH 04/10] Document the aggregate-impact report and mandatory test data Refs #2334. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/testing.md | 16 +++ .../utils/aggregate_impact/README.md | 102 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 policyengine_uk/utils/aggregate_impact/README.md diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 66cb54d3c4..9657daece1 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -25,6 +25,22 @@ Use this skill whenever adding, moving, or reviewing tests. private H5 files in unit-style tests. Mock those seams or skip cleanly when the external artifact is unavailable. +## Microsimulation Data + +Microsimulation tests run on one certified dataset, pinned by Hugging Face +commit and SHA-256 in `policyengine_uk/tests/certified_dataset.yaml`. With +`HUGGING_FACE_TOKEN` set, `conftest.py` selects that dataset and verifies its +bytes before the first data test runs. Without a token, data tests skip. + +Set `POLICYENGINE_UK_REQUIRE_TEST_DATA=1` in jobs that must run the data tests. +Each data test then fails at setup when the dataset is unavailable, instead of +skipping, and runs only on the pinned dataset. Credential-free tests are +unaffected. + +The reform fixtures in `microsimulation/reforms_config.yaml` record the dataset +they were computed on. When the pin moves, recompute them on the new dataset, +and add each new value under `receipts` rather than overwriting the old ones. + ## Commands Run lint and formatting before committing: diff --git a/policyengine_uk/utils/aggregate_impact/README.md b/policyengine_uk/utils/aggregate_impact/README.md new file mode 100644 index 0000000000..0a9c73bfea --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/README.md @@ -0,0 +1,102 @@ +# Aggregate impact report + +This tool shows what a change does to national numbers. It runs the model on +the certified dataset for two checkouts (a pull request's base and its head) +and compares them. Each side runs every year from 2025-26 to 2031-32 as a full +microsimulation. + +The report covers: +- receipts by tax head; +- spending and caseloads by benefit; +- poverty; +- inequality; +- deciles; +- the standard reform costings. + +The `Aggregate impact` workflow (`.github/workflows/aggregate-impact.yaml`) +runs it on every pull request and keeps one comment on the PR up to date. + +## Running it + +```bash +export HUGGING_FACE_TOKEN=... # read access to policyengine-uk-data-private +uv run python -m policyengine_uk.utils.aggregate_impact measure --label head \ + --pin policyengine_uk/tests/certified_dataset.yaml \ + --reforms policyengine_uk/tests/microsimulation/reforms_config.yaml \ + --out out/head +# ... the same for a base checkout, into out/base, then: +uv run python -m policyengine_uk.utils.aggregate_impact compare \ + --base out/base --head out/head --markdown comment.md --summary summary.json +``` + +`measure` downloads the pinned file if it isn't cached, checks its SHA-256, and +writes `result.json` (figures, checks, provenance) and `households.npz` +(household detail for the decile comparison). If it crashes, it still writes a +`result.json` recording the crash, so the comparison can report it. + +To measure an older checkout with this version of the tool, copy this +directory under another name and run it in that checkout's environment, as CI +does. The tool imports only its own modules and `policyengine_uk`'s public +`Microsimulation`: + +```bash +cp -R policyengine_uk/utils/aggregate_impact /tmp/tool/pe_uk_aggregate_impact +cd ../base-checkout && PYTHONPATH=/tmp/tool .venv/bin/python -m pe_uk_aggregate_impact measure ... +``` + +For use from other code (qualification in policyengine.py), see `api.py`: +`compute_budget_metrics(sim, years=...)` returns a long-format report. + +## What fails and what is flagged + +The comparison **fails** in these cases: +- the head crashes; +- base and head ran on different dataset bytes; +- a figure is not finite; +- the head breaks a check that the base passes. + +A check that fails on both sides is reported as pre-existing and does not +fail the job; `--strict` makes it fail. Moves above the thresholds in +`report_spec.yaml` are **flagged** for review, never failed, because intended +policy changes move numbers. + +The checks are: +- **closure:** for each model total (`gov_tax`, `household_tax`, + `pre_budget_change_household_tax`, `gov_spending`, `household_benefits`), the + components in the model's own list are exactly the partition's heads and + declared double counts; +- **identities**, checked household by household: + - the heads and declared double counts add up to each total; + - `gov_balance` equals `gov_tax` minus `gov_spending`; + - household net income equals market income plus benefits, less taxes and + pension contributions; +- **counted once:** each component is raised by £1 per unit. Each tax total it + is in must rise by exactly £1 per unit, and each spending total by at most + £1, because means tests can claw the rise back but nothing should add to + it. A declared double count must still be contained in its head; +- **signs, bounds and finiteness**, and **determinism:** a rebuilt + simulation reproduces the baseline exactly; +- **inert heads:** a head that is zero for every unit is a warning, unless the + partition marks it `zero_at_baseline: expected`. + +## The fiscal partition + +`policyengine_uk/data/fiscal_partition.yaml` divides each total into heads, +each measured by one variable. Its header defines the fields. Each side of a +comparison reads its own partition, so a change to the model's totals and the +matching change to the partition go in the same pull request. + +When you add a component to one of the model's lists, add a head for it, or +the closure check fails. When the model stops counting something twice, +remove its `model_double_counts` entry and give it a head, or the +counted-once check fails as stale. + +## Years + +Model year 2026 is fiscal year 2026-27. The dataset is extended explicitly to +2030 (`extend_single_year_dataset`). Values for 2031-32 come from +policyengine-core instead: +- inputs with an `uprating` index are uprated from their 2030 values; +- other inputs are carried over unchanged. + +So treat 2031-32 figures as provisional. From afa1c2a1572feb8fe58408115218f6a611fdcf46 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:14:56 -0400 Subject: [PATCH 05/10] Name the measured refs readably in the aggregate-impact report Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/aggregate-impact.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/aggregate-impact.yaml b/.github/workflows/aggregate-impact.yaml index 9b0f14e2bc..a3899c6850 100644 --- a/.github/workflows/aggregate-impact.yaml +++ b/.github/workflows/aggregate-impact.yaml @@ -61,6 +61,8 @@ jobs: env: EVENT: ${{ github.event_name }} PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} INPUT_BASE: ${{ inputs.base }} INPUT_HEAD: ${{ inputs.head }} run: | @@ -68,15 +70,19 @@ jobs: # the base, so the comparison is exactly what merging would change. if [ "$EVENT" = "pull_request" ]; then base="$PR_BASE"; head="$GITHUB_SHA" + base_name="$PR_BASE_REF"; head_name="#$PR_NUMBER merged into $PR_BASE_REF" else base="${INPUT_BASE:-main}"; head="${INPUT_HEAD:-$GITHUB_SHA}" + base_name="$base"; head_name="${INPUT_HEAD:-$GITHUB_REF_NAME}" fi echo "base=$base" >> "$GITHUB_OUTPUT" echo "head=$head" >> "$GITHUB_OUTPUT" if [ "${{ matrix.side }}" = "base" ]; then echo "model=$base" >> "$GITHUB_OUTPUT" + echo "name=$base_name" >> "$GITHUB_OUTPUT" else echo "model=$head" >> "$GITHUB_OUTPUT" + echo "name=$head_name" >> "$GITHUB_OUTPUT" fi - name: Check out the head (tool, pin, partition fallback) @@ -110,7 +116,7 @@ jobs: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} ACTOR: ${{ github.actor }} SIDE: ${{ matrix.side }} - MODEL_SHA: ${{ steps.commits.outputs.model }} + MODEL_NAME: ${{ steps.commits.outputs.name }} run: | if [ -z "$HUGGING_FACE_TOKEN" ]; then if [ "$ACTOR" = "dependabot[bot]" ]; then @@ -133,7 +139,7 @@ jobs: --reforms ../tool-source/policyengine_uk/tests/microsimulation/reforms_config.yaml \ --fallback-partition ../tool-source/policyengine_uk/data/fiscal_partition.yaml \ --git-sha "$(git rev-parse HEAD)" \ - --git-ref "$MODEL_SHA" \ + --git-ref "$MODEL_NAME" \ --out "$GITHUB_WORKSPACE/out/$SIDE" status=$? set -e From 8d3e48204d2a02d4ee64a347fe8f74c980b4a0ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:24:20 -0400 Subject: [PATCH 06/10] Check that each standard reform moves its target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The standard reforms move into the tool (standard_reforms.yaml), so a vendored copy carries them: the eight reform-cost fixtures, plus a 10% rise in the Universal Credit standard allowance and in the State Pension. Each names the variable it must move, the direction and a minimum. A reform that moves its target by less, by nothing or the wrong way in any year fails responsive:, so a reform the model silently ignores cannot pass as a small number. A relative reform is built from each side's own baseline parameters: that model year's value times the factor, for every year of the horizon. On uk-data 1.58.0 all ten respond in every year, 2025-26 to 2031-32. The UC standard allowance rise adds £3.0bn to £3.4bn of Universal Credit, and the State Pension rise £12.4bn to £14.4bn. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/aggregate-impact.yaml | 1 - .../test_aggregate_impact_certified.py | 23 +++ .../test_aggregate_impact_reforms.py | 142 +++++++++++++++++ .../utils/aggregate_impact/README.md | 22 ++- .../utils/aggregate_impact/__main__.py | 22 +-- policyengine_uk/utils/aggregate_impact/api.py | 10 +- .../utils/aggregate_impact/compare.py | 1 + .../utils/aggregate_impact/measure.py | 111 ++++++++++--- .../utils/aggregate_impact/reforms.py | 148 ++++++++++++++++++ .../utils/aggregate_impact/render.py | 1 + .../aggregate_impact/standard_reforms.yaml | 106 +++++++++++++ 11 files changed, 545 insertions(+), 42 deletions(-) create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py create mode 100644 policyengine_uk/utils/aggregate_impact/reforms.py create mode 100644 policyengine_uk/utils/aggregate_impact/standard_reforms.yaml diff --git a/.github/workflows/aggregate-impact.yaml b/.github/workflows/aggregate-impact.yaml index a3899c6850..c582b243d4 100644 --- a/.github/workflows/aggregate-impact.yaml +++ b/.github/workflows/aggregate-impact.yaml @@ -136,7 +136,6 @@ jobs: PYTHONPATH="$tool" .venv/bin/python -m pe_uk_aggregate_impact measure \ --label "$SIDE" \ --pin ../tool-source/policyengine_uk/tests/certified_dataset.yaml \ - --reforms ../tool-source/policyengine_uk/tests/microsimulation/reforms_config.yaml \ --fallback-partition ../tool-source/policyengine_uk/data/fiscal_partition.yaml \ --git-sha "$(git rev-parse HEAD)" \ --git-ref "$MODEL_NAME" \ diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py index 062945b2b1..48af46e25d 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py @@ -14,6 +14,7 @@ default_partition_path, load_partition, ) +from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parameter_node from policyengine_uk.utils.aggregate_impact.spec import load_spec pytestmark = pytest.mark.microsimulation @@ -51,6 +52,7 @@ def measured(): partition, spec, label="test", + reforms=[r for r in load_reforms() if r.scale], guard_variables=tax_components | SPENDING_SAMPLE, ) return result, households, partition, Microsimulation(dataset=dataset) @@ -142,3 +144,24 @@ def test_household_detail_is_complete(measured): assert all(len(v) == n for v in columns.values()) assert np.all(np.isfinite(columns["income"])) assert set(np.unique(columns["decile"])) <= set(range(-1, 11)) + + +def test_relative_reforms_scale_their_parameters_and_move_their_targets(measured): + result, _, _, sim = measured + passing = {c["id"] for c in result["checks"] if c["status"] == "pass"} + relative = [r for r in load_reforms() if r.scale] + assert {r.id for r in relative} == { + "uc_standard_allowance_plus_10pc", + "state_pension_plus_10pc", + } + outcomes = {o["id"]: o for o in result["reforms"]} + for reform in relative: + assert f"reform_runs:{reform.id}" in passing + assert f"responsive:{reform.id}" in passing + for path, factor in reform.scale.items(): + baseline = parameter_node(sim.tax_benefit_system.parameters, path)( + f"{YEAR}-01-01" + ) + assert outcomes[reform.id]["parameters"][path][str(YEAR)] == pytest.approx( + baseline * factor + ) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py new file mode 100644 index 0000000000..ccccd7cec8 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py @@ -0,0 +1,142 @@ +"""The standard reform set and the responsiveness rule.""" + +from pathlib import Path + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem +from policyengine_uk.utils.aggregate_impact.reforms import ( + ReformSpecError, + Responds, + load_reforms, + parameter_node, + parse_reforms, + reform_parameters, + responsiveness, +) +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +TESTS = Path(__file__).resolve().parents[1] +YEARS = load_spec().years +STANDARD = load_reforms() + + +@pytest.fixture(scope="module") +def system(): + return CountryTaxBenefitSystem() + + +def test_every_standard_reform_says_what_it_must_move(system): + assert len(STANDARD) == 10 + assert len({r.id for r in STANDARD}) == len(STANDARD) + for reform in STANDARD: + assert reform.responds is not None, reform.id + assert reform.responds.variable in system.variables, reform.id + assert reform.responds.minimum > 0 + assert bool(reform.parameters) != bool(reform.scale), reform.id + + +def test_standard_set_contains_the_reform_cost_fixtures_unchanged(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f)["reforms"] + by_name = {r.name: r for r in STANDARD} + for fixture in fixtures: + assert dict(by_name[fixture["name"]].parameters) == fixture["parameters"] + + +def test_every_named_parameter_exists(system): + for reform in STANDARD: + for path in list(reform.parameters) + list(reform.scale): + assert parameter_node(system.parameters, path) is not None, path + + +def test_bracket_paths_resolve(system): + node = parameter_node(system.parameters, "gov.hmrc.income_tax.rates.uk[0].rate") + assert node("2026-01-01") == pytest.approx(0.2) + + +def test_scaled_reforms_are_the_baseline_times_the_factor_in_every_year(system): + for reform in STANDARD: + changes = reform_parameters(reform, system.parameters, YEARS) + for path, factor in reform.scale.items(): + assert set(changes[path]) == {str(y) for y in YEARS} + node = parameter_node(system.parameters, path) + for year in YEARS: + assert changes[path][str(year)] == pytest.approx( + node(f"{year}-01-01") * factor + ) + for path, value in reform.parameters.items(): + assert changes[path] == value + + +@pytest.mark.parametrize( + "entry", + [ + {"name": "no change"}, + { + "name": "x", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "sideways", "minimum": 1}, + }, + { + "name": "x", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "up", "minimum": 0}, + }, + {"name": "x", "scale": {"a": 0}}, + ], +) +def test_malformed_reforms_are_rejected(entry): + with pytest.raises(ReformSpecError): + parse_reforms({"reforms": [entry]}) + + +def test_duplicate_ids_are_rejected(): + entry = {"name": "Same name", "parameters": {"a": 1}} + with pytest.raises(ReformSpecError): + parse_reforms({"reforms": [entry, dict(entry)]}) + + +def test_fixture_schema_parses_without_responds(): + reforms = parse_reforms({"reforms": [{"name": "Raise X", "parameters": {"a": 1}}]}) + assert reforms[0].id == "raise_x" and reforms[0].responds is None + + +changes = st.dictionaries( + st.integers(2025, 2035), + st.floats(-1e11, 1e11, allow_nan=False), + min_size=1, + max_size=8, +) + + +@settings(max_examples=200, deadline=None) +@given(changes=changes, minimum=st.floats(1, 1e10)) +def test_responsive_exactly_when_every_year_moves_enough_the_right_way( + changes, minimum +): + up, _ = responsiveness(Responds("v", "up", minimum), changes) + assert up == all(c >= minimum for c in changes.values()) + # Mirror image: the same reform read as a fall of the negated changes. + down, _ = responsiveness( + Responds("v", "down", minimum), {y: -c for y, c in changes.items()} + ) + assert down == up + + +@settings(max_examples=100, deadline=None) +@given(changes=changes, minimum=st.floats(1, 1e10), year=st.integers(2025, 2035)) +def test_one_inert_or_wrong_signed_year_fails(changes, minimum, year): + for bad in (0.0, -abs(minimum)): + ok, detail = responsiveness( + Responds("v", "up", minimum), {**changes, year: bad} + ) + assert not ok and str(year)[2:] in detail + + +def test_no_measured_year_fails(): + ok, detail = responsiveness(Responds("v", "up", 1.0), {}) + assert not ok and "no year" in detail diff --git a/policyengine_uk/utils/aggregate_impact/README.md b/policyengine_uk/utils/aggregate_impact/README.md index 0a9c73bfea..a4b946777e 100644 --- a/policyengine_uk/utils/aggregate_impact/README.md +++ b/policyengine_uk/utils/aggregate_impact/README.md @@ -21,9 +21,7 @@ runs it on every pull request and keeps one comment on the PR up to date. ```bash export HUGGING_FACE_TOKEN=... # read access to policyengine-uk-data-private uv run python -m policyengine_uk.utils.aggregate_impact measure --label head \ - --pin policyengine_uk/tests/certified_dataset.yaml \ - --reforms policyengine_uk/tests/microsimulation/reforms_config.yaml \ - --out out/head + --pin policyengine_uk/tests/certified_dataset.yaml --out out/head # ... the same for a base checkout, into out/base, then: uv run python -m policyengine_uk.utils.aggregate_impact compare \ --base out/base --head out/head --markdown comment.md --summary summary.json @@ -77,7 +75,23 @@ The checks are: - **signs, bounds and finiteness**, and **determinism:** a rebuilt simulation reproduces the baseline exactly; - **inert heads:** a head that is zero for every unit is a warning, unless the - partition marks it `zero_at_baseline: expected`. + partition marks it `zero_at_baseline: expected`; +- **reform responsiveness:** each standard reform must move the variable it + targets in the stated direction, by at least its stated minimum, in every + year. + +## The standard reforms + +`standard_reforms.yaml` lists the reforms costed on each side: the eight +reform-cost fixtures, plus a 10% rise in the Universal Credit standard +allowance and a 10% rise in the State Pension. Each says what it must move +(`responds`). A reform that the model ignores, or that moves its target the +wrong way, fails `responsive:` instead of passing as a small number. + +A relative reform (`scale`) is built from each side's own baseline: every +listed parameter gets that model year's baseline value times the factor, for +each year of the horizon. Pass `--reforms` to cost a different file, or +`--no-reforms` to cost none. ## The fiscal partition diff --git a/policyengine_uk/utils/aggregate_impact/__main__.py b/policyengine_uk/utils/aggregate_impact/__main__.py index 999c9f006e..d3dbf60dee 100644 --- a/policyengine_uk/utils/aggregate_impact/__main__.py +++ b/policyengine_uk/utils/aggregate_impact/__main__.py @@ -1,9 +1,7 @@ """Command line: ``measure`` one checkout, then ``compare`` two measurements. python -m policyengine_uk.utils.aggregate_impact measure --label head \\ - --pin policyengine_uk/tests/certified_dataset.yaml \\ - --reforms policyengine_uk/tests/microsimulation/reforms_config.yaml \\ - --out out/head + --pin policyengine_uk/tests/certified_dataset.yaml --out out/head python -m policyengine_uk.utils.aggregate_impact compare \\ --base out/base --head out/head --markdown comment.md --summary summary.json @@ -20,8 +18,6 @@ import traceback from pathlib import Path -import yaml - from . import TOOL_SCHEMA @@ -79,11 +75,13 @@ def cmd_measure(args) -> int: partition = load_partition(partition_path) provenance["partition_source"] = partition_source + from .reforms import DEFAULT_REFORMS_PATH, load_reforms + reforms = [] - if args.reforms: - with open(args.reforms) as f: - reforms = (yaml.safe_load(f) or {}).get("reforms", []) - provenance["reforms_sha256"] = _sha256(args.reforms) + if not args.no_reforms: + reforms_path = Path(args.reforms) if args.reforms else DEFAULT_REFORMS_PATH + reforms = load_reforms(reforms_path) + provenance["reforms_sha256"] = _sha256(reforms_path) from policyengine_uk import Microsimulation @@ -167,7 +165,11 @@ def main(argv=None) -> int: ) m.add_argument("--spec", help="report spec YAML") m.add_argument("--years", help="override the horizon, e.g. 2025-2031") - m.add_argument("--reforms", help="reform regression YAML (a 'reforms' list)") + m.add_argument( + "--reforms", + help="reform YAML to cost instead of the standard set (standard_reforms.yaml)", + ) + m.add_argument("--no-reforms", action="store_true", help="cost no reforms") m.add_argument("--git-sha", default="") m.add_argument("--git-ref", default="") m.add_argument( diff --git a/policyengine_uk/utils/aggregate_impact/api.py b/policyengine_uk/utils/aggregate_impact/api.py index dc3058c14f..20445c05aa 100644 --- a/policyengine_uk/utils/aggregate_impact/api.py +++ b/policyengine_uk/utils/aggregate_impact/api.py @@ -21,6 +21,7 @@ from . import TOOL_SCHEMA from .measure import measure, tool_digest from .partition import Partition, default_partition_path, load_partition +from .reforms import load_reforms from .spec import DEFAULT_SPEC_PATH, Spec, load_spec KIND_BY_SECTION = { @@ -37,6 +38,7 @@ "household_totals": "household_total", "population": "population", "reforms": "reform", + "reform_targets": "reform_target", } PARTITIONED_SECTIONS = {"receipts", "spending", "caseloads"} @@ -103,7 +105,7 @@ def compute_budget_metrics( years: Optional[Sequence[int]] = None, partition: Optional[Partition] = None, spec: Optional[Spec] = None, - reforms: Sequence[dict] = (), + reforms: Optional[Sequence] = None, run_guard: bool = True, label: str = "candidate", provenance: Optional[dict] = None, @@ -111,7 +113,9 @@ def compute_budget_metrics( """Measure ``sim`` (a fresh ``Microsimulation``) and return the report. ``factory(reform=None)`` builds another simulation on the same data; by - default it rebuilds ``type(sim)`` from ``sim.dataset``. A failed identity + default it rebuilds ``type(sim)`` from ``sim.dataset``. ``reforms`` + defaults to the standard set (``standard_reforms.yaml``), each costed and + checked for responsiveness; pass ``[]`` to cost none. A failed identity has status ``fail``; ``warn`` rows (a head that is zero everywhere) are observations, not failures. """ @@ -130,6 +134,8 @@ def compute_budget_metrics( def factory(reform=None): return simulation_class(dataset=dataset, reform=reform or None) + if reforms is None: + reforms = load_reforms() provenance = dict(provenance or {}) provenance.setdefault("metrics_version", f"{TOOL_SCHEMA}+{tool_digest()[:16]}") result, _ = measure( diff --git a/policyengine_uk/utils/aggregate_impact/compare.py b/policyengine_uk/utils/aggregate_impact/compare.py index f529e457dd..b4052868f1 100644 --- a/policyengine_uk/utils/aggregate_impact/compare.py +++ b/policyengine_uk/utils/aggregate_impact/compare.py @@ -21,6 +21,7 @@ FLAG_KIND_BY_SECTION = { "reforms": "reform_gbp", + "reform_targets": "reform_gbp", } DECILES = tuple(range(1, 11)) diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py index 4f4080aecb..59e0482bb2 100644 --- a/policyengine_uk/utils/aggregate_impact/measure.py +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -18,6 +18,7 @@ from . import TOOL_SCHEMA from .guard import counted_once, snapshot_inputs from .partition import Partition, closure_problems, group_label, read_model_lists +from .reforms import StandardReform, parse_reforms, reform_parameters, responsiveness from .spec import Spec, fiscal_year_label ENTITIES = ("person", "benunit", "household") @@ -530,14 +531,35 @@ def _finite_metrics_check(rec: Recorder): ) +def weighted_total(sim, variable: str, year: int) -> float: + """The household-weighted total of a variable of any entity.""" + hw = array(sim, "household_weight", year).astype(np.float64) + values = array(sim, variable, year).astype(np.float64) + return float(np.sum(hw * to_household(sim, values, entity_of(sim, variable)))) + + +def as_standard_reforms(reforms) -> list: + """Accept parsed reforms or raw ``{name, parameters, ...}`` mappings.""" + reforms = list(reforms or []) + if all(isinstance(r, StandardReform) for r in reforms): + return reforms + return parse_reforms({"reforms": reforms}) + + def measure_reforms( factory: Callable, + baseline_parameters, baseline_balance: Mapping[int, float], - reforms: Sequence[Mapping], + baseline_targets: Mapping[int, Mapping[str, float]], + reforms: Sequence[StandardReform], years: Sequence[int], rec: Recorder, ) -> list: - """Each reform's change in gov_balance against the baseline run, by year. + """Cost each reform against the baseline run, and check it responds. + + Records each reform's change in gov_balance by year and, where the + reform declares what it must move, the change in that variable's total + and whether it moved enough in the right direction every year. First rebuilds an unreformed simulation the same way, which must reproduce the baseline exactly: otherwise reform costings would include @@ -547,10 +569,7 @@ def measure_reforms( first = years[0] try: fresh = factory(reform=None) - hw = array(fresh, "household_weight", first).astype(np.float64) - again = float( - np.sum(hw * array(fresh, "gov_balance", first).astype(np.float64)) - ) + again = weighted_total(fresh, "gov_balance", first) rec.check( "determinism:rebuilt_baseline", "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", @@ -569,32 +588,65 @@ def measure_reforms( ) gc.collect() for reform in reforms: - name = str(reform["name"]) - reform_id = slug(name) outcome = { - "id": reform_id, - "name": name, - "parameters": reform.get("parameters", {}), + "id": reform.id, + "name": reform.name, + "parameters": None, + "responds": None, "error": None, } + target_changes = {} try: - reformed = factory(reform=reform.get("parameters")) + parameters = reform_parameters(reform, baseline_parameters, years) + outcome["parameters"] = parameters + reformed = factory(reform=parameters) for year in years: - hw = array(reformed, "household_weight", year).astype(np.float64) - balance = float( - np.sum(hw * array(reformed, "gov_balance", year).astype(np.float64)) - ) rec.metric( - f"reforms.{reform_id}", - name, + f"reforms.{reform.id}", + reform.name, "reforms", "gbp", year, - balance - baseline_balance[year], + weighted_total(reformed, "gov_balance", year) + - baseline_balance[year], ) + if reform.responds is not None: + variable = reform.responds.variable + change = ( + weighted_total(reformed, variable, year) + - baseline_targets[year][variable] + ) + target_changes[year] = change + rec.metric( + f"reform_targets.{reform.id}", + f"{reform.name}: change in {variable}", + "reform_targets", + "gbp", + year, + change, + ) del reformed except Exception as error: # noqa: BLE001 - one reform's failure is reported, not fatal outcome["error"] = f"{type(error).__name__}: {error}" + if reform.responds is not None: + outcome["responds"] = { + "variable": reform.responds.variable, + "direction": reform.responds.direction, + "minimum": reform.responds.minimum, + } + if outcome["error"] is None: + ok, detail = responsiveness(reform.responds, target_changes) + else: + ok, detail = False, "the reform did not run" + rec.check( + f"responsive:{reform.id}", + f"'{reform.name}' moves {reform.responds.variable} " + f"{reform.responds.direction} by at least " + f"£{reform.responds.minimum / 1e9:,.2f}bn in every year", + ok, + detail, + section="reforms", + ) gc.collect() outcomes.append(outcome) return outcomes @@ -633,16 +685,19 @@ def measure( ) snapshot = snapshot_inputs(sim, partition.component_variables()) + reforms = as_standard_reforms(reforms) + target_variables = sorted({r.responds.variable for r in reforms if r.responds}) households = {} - baseline_balance = {} + baseline_balance, baseline_targets = {}, {} reset = getattr(sim, "_invalidate_all_caches", None) for year in spec.years: t = time.time() measure_year(sim, partition, spec, year, rec, households) - hw = array(sim, "household_weight", year).astype(np.float64) - baseline_balance[year] = float( - np.sum(hw * array(sim, "gov_balance", year).astype(np.float64)) - ) + baseline_balance[year] = weighted_total(sim, "gov_balance", year) + baseline_targets[year] = { + variable: weighted_total(sim, variable, year) + for variable in target_variables + } timings[f"year_{year}"] = round(time.time() - t, 2) if reset is not None: # Keep memory flat across the horizon; supplied inputs survive. @@ -702,7 +757,13 @@ def measure( t = time.time() reform_outcomes = measure_reforms( - factory, baseline_balance, reforms, spec.years, rec + factory, + sim.tax_benefit_system.parameters, + baseline_balance, + baseline_targets, + reforms, + spec.years, + rec, ) timings["reforms"] = round(time.time() - t, 2) for outcome in reform_outcomes: diff --git a/policyengine_uk/utils/aggregate_impact/reforms.py b/policyengine_uk/utils/aggregate_impact/reforms.py new file mode 100644 index 0000000000..c5ec5a77b8 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/reforms.py @@ -0,0 +1,148 @@ +"""The standard reform set: loading, building reforms, and responsiveness.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Optional, Sequence + +import yaml + +from .spec import fiscal_year_label + +DEFAULT_REFORMS_PATH = Path(__file__).with_name("standard_reforms.yaml") +_INDEXED = re.compile(r"^(?P[^\[\]]+)\[(?P\d+)\]$") + + +class ReformSpecError(ValueError): + pass + + +@dataclass(frozen=True) +class Responds: + """What a reform must do to the weighted total of one model variable.""" + + variable: str + direction: str # "up" or "down" + minimum: float + + +@dataclass(frozen=True) +class StandardReform: + id: str + name: str + parameters: Mapping = field(default_factory=dict) + scale: Mapping = field(default_factory=dict) + responds: Optional[Responds] = None + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def parse_reforms(raw: Mapping) -> list: + """Parse a reform file. + + Accepts the standard schema and the reform-cost fixture schema (a + ``reforms`` list of ``name`` and ``parameters``, with no ``responds``). + """ + reforms, seen = [], set() + for index, entry in enumerate((raw or {}).get("reforms") or []): + where = f"reforms[{index}]" + if not isinstance(entry, Mapping) or not entry.get("name"): + raise ReformSpecError(f"{where}: needs a name") + parameters = dict(entry.get("parameters") or {}) + scale = {str(k): float(v) for k, v in (entry.get("scale") or {}).items()} + if not parameters and not scale: + raise ReformSpecError(f"{where}: needs parameters or scale") + for path, factor in scale.items(): + if not factor > 0: + raise ReformSpecError( + f"{where}: scale factor for {path} must be positive" + ) + responds = None + if entry.get("responds"): + r = entry["responds"] + if r.get("direction") not in ("up", "down"): + raise ReformSpecError(f"{where}: responds.direction must be up or down") + minimum = float(r.get("minimum", 0)) + if not r.get("variable") or not minimum > 0: + raise ReformSpecError( + f"{where}: responds needs a variable and a positive minimum" + ) + responds = Responds(str(r["variable"]), r["direction"], minimum) + reform_id = str(entry.get("id") or _slug(str(entry["name"]))) + if reform_id in seen: + raise ReformSpecError(f"{where}: reform id {reform_id!r} is used twice") + seen.add(reform_id) + reforms.append( + StandardReform(reform_id, str(entry["name"]), parameters, scale, responds) + ) + return reforms + + +def load_reforms(path=None) -> list: + with open(path or DEFAULT_REFORMS_PATH) as f: + return parse_reforms(yaml.safe_load(f)) + + +def parameter_node(root, path: str): + """Navigate ``a.b[0].c`` from a parameter tree's root.""" + node = root + for part in path.split("."): + indexed = _INDEXED.match(part) + if indexed: + node = getattr(node, indexed["name"]).brackets[int(indexed["index"])] + else: + node = getattr(node, part) + return node + + +def reform_parameters( + reform: StandardReform, baseline_parameters, years: Sequence[int] +) -> dict: + """The parameter-change dict for ``Microsimulation(reform=...)``. + + A scaled parameter gets one bare-year entry per horizon year: the + baseline's value for that model year times the factor. On this route a + bare year sets that model year alone. + """ + changes = dict(reform.parameters) + for path, factor in reform.scale.items(): + node = parameter_node(baseline_parameters, path) + changes[path] = { + str(year): float(node(f"{year}-01-01")) * factor for year in years + } + return changes + + +def responsiveness(responds: Responds, changes: Mapping[int, float]) -> tuple: + """Whether a reform moved its target enough, the right way, in every year. + + ``changes`` maps year to the reformed total minus the baseline total. + Returns ``(ok, detail)``; ``detail`` names the failing years. + """ + sign = 1.0 if responds.direction == "up" else -1.0 + failing = [] + for year, change in sorted(changes.items()): + moved = sign * change + if not moved >= responds.minimum: + if change == 0: + what = "did not move" + elif moved < 0: + what = f"moved the wrong way (£{change:,.0f})" + else: + what = f"moved only £{change:,.0f}" + failing.append( + f"{fiscal_year_label(int(year))}: {responds.variable} {what}" + ) + if not changes: + failing.append("no year was measured") + detail = "" + if failing: + detail = ( + f"expected {responds.variable} {responds.direction} by at least " + f"£{responds.minimum:,.0f} a year; " + "; ".join(failing) + ) + return not failing, detail diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py index 374e4f7722..8924e15a2b 100644 --- a/policyengine_uk/utils/aggregate_impact/render.py +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -22,6 +22,7 @@ ("inequality", "Inequality"), ("deciles", "Mean household net income in each side's own deciles"), ("reforms", "Standard reform costings (change in gov_balance)"), + ("reform_targets", "Standard reforms: change in each reform's target"), ) MAX_FLAG_ROWS = 40 diff --git a/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml b/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml new file mode 100644 index 0000000000..c2447136de --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml @@ -0,0 +1,106 @@ +# The standard reforms the aggregate-impact report costs on each side. +# +# Each reform is either absolute (`parameters`: values applied from 2023, as +# the reform-cost fixtures in tests/microsimulation/reforms_config.yaml do) or +# relative (`scale`: each listed parameter is multiplied by the factor in +# every year of the horizon, starting from that year's baseline value). +# +# `responds` states what the reform must do: the model variable whose +# weighted total it must move, the direction, and the least it must move it +# in every year of the horizon. A reform that moves its target by less, by +# nothing or the wrong way fails the `responsive:` check, so a reform the +# model silently ignores cannot pass as a small number. Each minimum is well +# below the reform's smallest yearly effect on uk-data 1.58.0 in October 2026 +# (about a fifth of it for most), and is a proposal awaiting the +# numerical-acceptance ruling (cos d1344). An absolute reform shrinks as the +# baseline is uprated towards it: the 13,000 personal allowance is worth £4.4bn +# in 2030-31 but £1.8bn in 2031-32, when the baseline allowance rises to +# 12,830. From 2032-33 the baseline (13,090) exceeds it, so before the horizon +# reaches that year, make that reform relative. +schema_version: 1 + +reforms: + - id: basic_rate_plus_1pp + name: Raise basic rate by 1pp + parameters: + gov.hmrc.income_tax.rates.uk[0].rate: 0.21 + responds: + variable: income_tax + direction: up + minimum: 1500000000 + - id: higher_rate_plus_1pp + name: Raise higher rate by 1pp + parameters: + gov.hmrc.income_tax.rates.uk[1].rate: 0.42 + responds: + variable: income_tax + direction: up + minimum: 1000000000 + - id: personal_allowance_13000 + name: Raise personal allowance by ~800GBP/year + parameters: + gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 + responds: + variable: income_tax + direction: down + minimum: 1000000000 + - id: child_benefit_additional_25 + name: Raise child benefit by 25GBP/week per additional child + parameters: + gov.hmrc.child_benefit.amount.additional: 25 + responds: + variable: child_benefit + direction: up + minimum: 300000000 + - id: uc_taper_20pc + name: Reduce Universal Credit taper rate to 20% + parameters: + gov.dwp.universal_credit.means_test.reduction_rate: 0.2 + responds: + variable: universal_credit + direction: up + minimum: 4000000000 + - id: employee_nics_main_10pc + name: Raise Class 1 main employee NICs rate to 10% + parameters: + gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 + responds: + variable: national_insurance + direction: up + minimum: 2500000000 + - id: vat_standard_plus_2pp + name: Raise VAT standard rate by 2pp + parameters: + gov.hmrc.vat.standard_rate: 0.22 + responds: + variable: vat + direction: up + minimum: 4000000000 + - id: additional_rate_plus_3pp + name: Raise additional rate by 3pp + parameters: + gov.hmrc.income_tax.rates.uk[2].rate: 0.48 + responds: + variable: income_tax + direction: up + minimum: 750000000 + - id: uc_standard_allowance_plus_10pc + name: Raise the Universal Credit standard allowance by 10% + scale: + gov.dwp.universal_credit.standard_allowance.amount.SINGLE_YOUNG: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.SINGLE_OLD: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.COUPLE_YOUNG: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.COUPLE_OLD: 1.1 + responds: + variable: universal_credit + direction: up + minimum: 600000000 + - id: state_pension_plus_10pc + name: Raise the State Pension by 10% + scale: + gov.dwp.state_pension.new_state_pension.amount: 1.1 + gov.dwp.state_pension.basic_state_pension.amount: 1.1 + responds: + variable: state_pension + direction: up + minimum: 2500000000 From fd944159ee27bd1a0f6a664bf537cb98d4b976ae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:28:49 -0400 Subject: [PATCH 07/10] Keep household-level detail on the runner The certified dataset is licensed survey microdata, and only aggregates may be published. The workflow uploaded each side's measurement as an artifact, including the household-level arrays the decile comparison uses, and cached the dataset in the Actions cache. Both are removed: - base and head are now measured in one job, so the household detail never travels between jobs; only the report directory (the two result.json files, the comment and the summary) is uploaded; - the dataset is downloaded on each run and never cached; - measure writes household detail under private/, apart from result.json; - check details give counts and weighted totals, not one household's value. test_aggregate_impact_workflow.py fails if the workflow uploads anything but the report, uses the Actions cache, copies anything private into the report, or splits the measurement across jobs. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/aggregate-impact.yaml | 167 +++++++----------- .../test_aggregate_impact_certified.py | 21 +++ .../test_aggregate_impact_workflow.py | 82 +++++++++ .../utils/aggregate_impact/README.md | 15 +- .../utils/aggregate_impact/measure.py | 35 +++- 5 files changed, 207 insertions(+), 113 deletions(-) create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py diff --git a/.github/workflows/aggregate-impact.yaml b/.github/workflows/aggregate-impact.yaml index c582b243d4..b3679d74dd 100644 --- a/.github/workflows/aggregate-impact.yaml +++ b/.github/workflows/aggregate-impact.yaml @@ -6,9 +6,16 @@ # (policyengine_uk/utils/aggregate_impact), each in an environment synced # from its own uv.lock, on the dataset pinned in # policyengine_uk/tests/certified_dataset.yaml (bytes verified on each side). -# Each side reads its own fiscal partition. The report job fails on a crash, -# a different dataset, non-finite figures or a check the head breaks that the +# Each side reads its own fiscal partition. The job fails on a crash, a +# different dataset, non-finite figures or a check the head breaks that the # base passes; moves above the flag thresholds are listed, not failed. +# +# DATA PROTECTION. The dataset is licensed survey microdata. Only aggregates +# may leave this job: the two result.json files, the comment and the summary. +# Never upload the measurement's private/ directory (household-level detail +# for the decile comparison), and never put the dataset in the Actions cache. +# Both sides are measured in one job so that detail stays on the runner. +# test_aggregate_impact_workflow.py enforces this. name: Aggregate impact @@ -42,19 +49,18 @@ env: PYTHONUNBUFFERED: "1" jobs: - measure: - name: Measure ${{ matrix.side }} + report: + name: Aggregate impact # Fork pull requests cannot read the private certified dataset. if: >- github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - # The same runner image for both sides, as for the Test job. runs-on: ubuntu-24.04-arm - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - side: [base, head] + timeout-minutes: 60 + permissions: + contents: read + issues: write + pull-requests: write steps: - name: Choose the commits id: commits @@ -75,55 +81,47 @@ jobs: base="${INPUT_BASE:-main}"; head="${INPUT_HEAD:-$GITHUB_SHA}" base_name="$base"; head_name="${INPUT_HEAD:-$GITHUB_REF_NAME}" fi - echo "base=$base" >> "$GITHUB_OUTPUT" - echo "head=$head" >> "$GITHUB_OUTPUT" - if [ "${{ matrix.side }}" = "base" ]; then - echo "model=$base" >> "$GITHUB_OUTPUT" - echo "name=$base_name" >> "$GITHUB_OUTPUT" - else - echo "model=$head" >> "$GITHUB_OUTPUT" - echo "name=$head_name" >> "$GITHUB_OUTPUT" - fi + { + echo "base=$base" + echo "head=$head" + echo "base_name=$base_name" + echo "head_name=$head_name" + } >> "$GITHUB_OUTPUT" - - name: Check out the head (tool, pin, partition fallback) + - name: Check out the head uses: actions/checkout@v6 with: ref: ${{ steps.commits.outputs.head }} - path: tool-source + path: head - - name: Check out the ${{ matrix.side }} model + - name: Check out the base uses: actions/checkout@v6 with: - ref: ${{ steps.commits.outputs.model }} - path: model + ref: ${{ steps.commits.outputs.base }} + path: base - name: Install uv uses: astral-sh/setup-uv@v8.1.0 - - name: Sync the ${{ matrix.side }} environment from its lock - working-directory: model - run: uv sync --frozen --no-dev - - - name: Restore the certified dataset - uses: actions/cache@v4 - with: - path: ~/.cache/huggingface/hub/models--policyengine--policyengine-uk-data-private - key: certified-data-${{ hashFiles('tool-source/policyengine_uk/tests/certified_dataset.yaml') }} + - name: Sync each side's environment from its own lock + run: | + (cd base && uv sync --frozen --no-dev) + (cd head && uv sync --frozen --no-dev) - - name: Measure - working-directory: model + - name: Measure base, then head + id: measure env: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} ACTOR: ${{ github.actor }} - SIDE: ${{ matrix.side }} - MODEL_NAME: ${{ steps.commits.outputs.name }} + BASE_NAME: ${{ steps.commits.outputs.base_name }} + HEAD_NAME: ${{ steps.commits.outputs.head_name }} run: | + mkdir -p measured/base measured/head if [ -z "$HUGGING_FACE_TOKEN" ]; then if [ "$ACTOR" = "dependabot[bot]" ]; then echo "::notice::Dependabot runs have no data access; the aggregate impact report is skipped." - mkdir -p "$GITHUB_WORKSPACE/out/$SIDE" echo '{"schema": "policyengine-uk-aggregate-impact/1", "status": "skipped", "reason": "Dependabot runs have no access to the certified dataset"}' \ - > "$GITHUB_WORKSPACE/out/$SIDE/result.json" + > measured/head/result.json exit 0 fi echo "::error::HUGGING_FACE_TOKEN is not available, so the certified dataset cannot be read." @@ -131,85 +129,48 @@ jobs: fi tool="$RUNNER_TEMP/tool" mkdir -p "$tool" - cp -R ../tool-source/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" - set +e - PYTHONPATH="$tool" .venv/bin/python -m pe_uk_aggregate_impact measure \ - --label "$SIDE" \ - --pin ../tool-source/policyengine_uk/tests/certified_dataset.yaml \ - --fallback-partition ../tool-source/policyengine_uk/data/fiscal_partition.yaml \ - --git-sha "$(git rev-parse HEAD)" \ - --git-ref "$MODEL_NAME" \ - --out "$GITHUB_WORKSPACE/out/$SIDE" - status=$? - set -e - # A crash is recorded in result.json for the report to judge; only - # a run that wrote nothing fails here. - if [ ! -f "$GITHUB_WORKSPACE/out/$SIDE/result.json" ]; then - echo "::error::The measurement wrote no result (exit $status)." + cp -R head/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + measure() { + side="$1"; name="$2" + # A crash is recorded in result.json for the comparison to judge. + (cd "$side" && PYTHONPATH="$tool" .venv/bin/python -m pe_uk_aggregate_impact measure \ + --label "$side" \ + --pin "$GITHUB_WORKSPACE/head/policyengine_uk/tests/certified_dataset.yaml" \ + --fallback-partition "$GITHUB_WORKSPACE/head/policyengine_uk/data/fiscal_partition.yaml" \ + --git-sha "$(git rev-parse HEAD)" \ + --git-ref "$name" \ + --out "$GITHUB_WORKSPACE/measured/$side") || true + } + measure base "$BASE_NAME" + measure head "$HEAD_NAME" + if [ ! -f measured/head/result.json ]; then + echo "::error::The head measurement wrote no result." exit 1 fi - - name: Upload the ${{ matrix.side }} measurement - if: always() - uses: actions/upload-artifact@v4 - with: - name: aggregate-impact-${{ matrix.side }} - path: out/${{ matrix.side }} - if-no-files-found: ignore - retention-days: 30 - - report: - name: Report - needs: measure - if: >- - always() && ( - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - ) - runs-on: ubuntu-latest - timeout-minutes: 15 - permissions: - contents: read - issues: write - pull-requests: write - steps: - - name: Check out the head - uses: actions/checkout@v6 - with: - ref: ${{ github.event_name == 'pull_request' && github.sha || inputs.head || github.sha }} - - - name: Install uv - uses: astral-sh/setup-uv@v8.1.0 - - - name: Download the measurements - uses: actions/download-artifact@v4 - with: - pattern: aggregate-impact-* - path: out - - name: Compare base with head id: compare run: | - tool="$RUNNER_TEMP/tool" - mkdir -p "$tool" - cp -R policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" - uv venv -q "$RUNNER_TEMP/compare-env" - uv pip install -q --python "$RUNNER_TEMP/compare-env" numpy pyyaml + mkdir -p report/base report/head base_args=() - if [ -f out/aggregate-impact-base/result.json ]; then - base_args=(--base out/aggregate-impact-base) + if [ -f measured/base/result.json ]; then + base_args=(--base measured/base) + cp measured/base/result.json report/base/result.json fi - mkdir -p out/aggregate-impact-head report + cp measured/head/result.json report/head/result.json set +e - PYTHONPATH="$tool" "$RUNNER_TEMP/compare-env/bin/python" -m pe_uk_aggregate_impact compare \ + PYTHONPATH="$RUNNER_TEMP/tool" head/.venv/bin/python -m pe_uk_aggregate_impact compare \ "${base_args[@]}" \ - --head out/aggregate-impact-head \ + --head measured/head \ --markdown report/comment.md \ --summary report/summary.json \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" echo "status=$?" >> "$GITHUB_OUTPUT" set -e cat report/comment.md >> "$GITHUB_STEP_SUMMARY" + # Household-level detail is no longer needed; remove it before any + # later step could pick it up. + rm -rf measured - name: Post or update the pull request comment if: github.event_name == 'pull_request' @@ -227,7 +188,7 @@ jobs: gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@report/comment.md > /dev/null fi - - name: Upload the report + - name: Upload the report (aggregates only) if: always() uses: actions/upload-artifact@v4 with: diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py index 48af46e25d..abce2bb4ce 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py @@ -165,3 +165,24 @@ def test_relative_reforms_scale_their_parameters_and_move_their_targets(measured assert outcomes[reform.id]["parameters"][path][str(YEAR)] == pytest.approx( baseline * factor ) + + +def test_the_published_result_holds_aggregates_only(measured): + """result.json may be public: no household-length arrays, no single values.""" + result, households, _, _ = measured + n_households = len(households[str(YEAR)]["household_id"]) + + def numeric_lists(node): + if isinstance(node, dict): + for value in node.values(): + yield from numeric_lists(value) + elif isinstance(node, list): + if node and all(isinstance(v, (int, float)) for v in node): + yield node + for value in node: + yield from numeric_lists(value) + + longest = max((len(values) for values in numeric_lists(result)), default=0) + assert longest <= 50 < n_households + details = " ".join(d for c in result["checks"] for d in c["details"]) + assert "minimum £" not in details and "largest gap" not in details diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py new file mode 100644 index 0000000000..0409d9c1a1 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py @@ -0,0 +1,82 @@ +"""Data protection: only aggregates may leave the aggregate-impact job. + +The certified dataset is licensed survey microdata. The measurement keeps +household-level detail for the decile comparison under ``private/``; these +tests fail if the workflow could publish it, or could put the dataset where +other workflows can read it. +""" + +import json +from pathlib import Path + +import numpy as np +import yaml + +from policyengine_uk.utils.aggregate_impact.measure import ( + HOUSEHOLDS_FILE, + PRIVATE_DIRECTORY, + read_result, + write_result, +) + +REPO = Path(__file__).resolve().parents[3] +WORKFLOW = REPO / ".github" / "workflows" / "aggregate-impact.yaml" +ALLOWED_UPLOAD_PATHS = {"report"} + + +def steps(): + workflow = yaml.safe_load(WORKFLOW.read_text()) + return [step for job in workflow["jobs"].values() for step in job["steps"]] + + +def test_only_the_aggregate_report_is_uploaded(): + uploads = [s for s in steps() if "upload-artifact" in str(s.get("uses", ""))] + assert uploads, "the report should be uploaded" + for step in uploads: + paths = {p.strip() for p in str(step["with"]["path"]).splitlines() if p.strip()} + assert paths <= ALLOWED_UPLOAD_PATHS, paths + + +def test_the_dataset_is_never_cached(): + for step in steps(): + assert "actions/cache" not in str(step.get("uses", "")), step.get("name") + + +def test_nothing_private_is_copied_into_the_report(): + text = WORKFLOW.read_text() + copies = [ + line.strip() for line in text.splitlines() if line.strip().startswith("cp ") + ] + for line in copies: + if "report/" in line: + assert line.split()[1].endswith("result.json"), line + assert f"{PRIVATE_DIRECTORY}/" not in "\n".join(copies) + assert HOUSEHOLDS_FILE not in "\n".join(copies) + + +def test_the_job_runs_both_sides_itself(): + workflow = yaml.safe_load(WORKFLOW.read_text()) + assert list(workflow["jobs"]) == ["report"], ( + "base and head must be measured in one job, so that household detail " + "never has to travel between jobs as an artifact" + ) + assert not [s for s in steps() if "download-artifact" in str(s.get("uses", ""))] + + +def test_household_detail_is_written_only_under_private(tmp_path): + households = { + "2026": { + "household_id": np.arange(5), + "household_weight": np.ones(5), + "household_count_people": np.ones(5), + "decile": np.arange(5), + "income": np.linspace(1, 2, 5), + } + } + write_result(tmp_path, {"status": "ok", "metrics": []}, households) + top_level = sorted(p.name for p in tmp_path.iterdir()) + assert top_level == [PRIVATE_DIRECTORY, "result.json"] + assert (tmp_path / PRIVATE_DIRECTORY / HOUSEHOLDS_FILE).exists() + result, loaded = read_result(tmp_path) + assert json.loads((tmp_path / "result.json").read_text()) == result + assert np.array_equal(loaded["2026"]["income"], households["2026"]["income"]) diff --git a/policyengine_uk/utils/aggregate_impact/README.md b/policyengine_uk/utils/aggregate_impact/README.md index a4b946777e..051597e470 100644 --- a/policyengine_uk/utils/aggregate_impact/README.md +++ b/policyengine_uk/utils/aggregate_impact/README.md @@ -16,6 +16,17 @@ The report covers: The `Aggregate impact` workflow (`.github/workflows/aggregate-impact.yaml`) runs it on every pull request and keeps one comment on the PR up to date. +## Data protection + +The certified dataset is licensed survey microdata. Only aggregates may be +published: `result.json`, the comment and the comparison summary. `measure` +also writes household-level detail for the decile comparison to +`private/households.npz`. Never upload, attach, commit or share anything +under `private/`, and never put the dataset in a shared cache. The workflow +measures both sides in one job so that this detail stays on the runner, and +`test_aggregate_impact_workflow.py` fails if the workflow could publish it. +Check details give counts and weighted totals, never one household's value. + ## Running it ```bash @@ -28,8 +39,8 @@ uv run python -m policyengine_uk.utils.aggregate_impact compare \ ``` `measure` downloads the pinned file if it isn't cached, checks its SHA-256, and -writes `result.json` (figures, checks, provenance) and `households.npz` -(household detail for the decile comparison). If it crashes, it still writes a +writes `result.json` (figures, checks, provenance) and +`private/households.npz` (household detail for the decile comparison). If it crashes, it still writes a `result.json` recording the crash, so the comparison can report it. To measure an older checkout with this version of the tool, copy this diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py index 59e0482bb2..73746112fb 100644 --- a/policyengine_uk/utils/aggregate_impact/measure.py +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -178,8 +178,12 @@ def version(name): } -def _identity_check(rec, check_id, label, components_hh, total_hh, year, spec): - """Household-level accounting identity: components add up to the total.""" +def _identity_check(rec, check_id, label, components_hh, total_hh, year, spec, weights): + """Household-level accounting identity: components add up to the total. + + The detail gives counts and weighted totals only: results may be public, + so they never quote one household's value. + """ difference = components_hh - total_hh scale = np.abs(total_hh) + 1.0 tolerance = 1.0 + spec.identity_relative_tolerance * scale @@ -188,10 +192,10 @@ def _identity_check(rec, check_id, label, components_hh, total_hh, year, spec): ok = bool(finite and not bad.any()) detail = "" if not ok: - worst = int(np.nanargmax(np.abs(difference))) if difference.size else 0 + gap = float(np.nansum(weights * np.where(bad, difference, 0.0))) detail = ( - f"{fiscal_year_label(year)}: {int(bad.sum())} households differ; " - f"largest gap £{float(difference[worst]):,.2f}" + f"{fiscal_year_label(year)}: {int(bad.sum())} households differ, " + f"by a weighted £{gap:,.0f} in total" ) rec.check(check_id, label, ok, detail, section="identities") @@ -259,7 +263,7 @@ def measure_year( f"{head.label} is never negative", not bool(negative.any()), f"{fiscal_year_label(year)}: {int(negative.sum())} units below zero, " - f"minimum £{float(np.nanmin(values)):,.2f}", + f"a weighted £{float(np.sum(hw * np.where(hh < 0, hh, 0.0))):,.0f} in total", section="signs", ) if not head.zero_at_baseline: @@ -311,6 +315,7 @@ def measure_year( hh, year, spec, + hw, ) def partition_sum(total): @@ -377,6 +382,7 @@ def partition_sum(total): gov_balance, year, spec, + hw, ) for variable, label in HOUSEHOLD_TOTALS.items(): if variable in tbs_variables: @@ -415,6 +421,7 @@ def partition_sum(total): array(sim, "household_net_income", year).astype(np.float64), year, spec, + hw, ) pw = weights["person"] @@ -793,7 +800,17 @@ def measure( return result, households +PRIVATE_DIRECTORY = "private" +HOUSEHOLDS_FILE = "households.npz" + + def write_result(out_dir, result: Mapping, households: Optional[Mapping] = None): + """Write the aggregates to result.json and household detail to private/. + + ``result.json`` holds only aggregates and may be published. Everything + under ``private/`` is household-level detail from licensed microdata: it + exists for the decile comparison and must never be uploaded or shared. + """ out_dir = Path(out_dir) out_dir.mkdir(parents=True, exist_ok=True) with open(out_dir / "result.json", "w") as f: @@ -804,7 +821,9 @@ def write_result(out_dir, result: Mapping, households: Optional[Mapping] = None) for year, columns in households.items() for name, values in columns.items() } - np.savez_compressed(out_dir / "households.npz", **arrays) + private = out_dir / PRIVATE_DIRECTORY + private.mkdir(exist_ok=True) + np.savez_compressed(private / HOUSEHOLDS_FILE, **arrays) def read_result(out_dir) -> tuple: @@ -816,7 +835,7 @@ def read_result(out_dir) -> tuple: with open(path) as f: result = json.load(f) households = None - npz = out_dir / "households.npz" + npz = out_dir / PRIVATE_DIRECTORY / HOUSEHOLDS_FILE if npz.exists(): households = {} with np.load(npz) as data: From c7a6c963991b4873a51042bda640f28eb914e34e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:44:55 -0400 Subject: [PATCH 08/10] Close the gaps the independent review found in the report's checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Counted-once check. It judged each total's net movement, which a means-test fall, or the eligibility-jump exclusion, could use to hide a double count. It now reads each head's response household by household: - a component's own head must rise by exactly £1; - no other tax head may move; - no other benefit may rise in any household (rises and falls are not netted); - the model's own total must move by what its heads moved, plus one for a declared double count. That last rule catches a total whose formula adds a zero component twice. A head that declares a read may respond, but not one for one. The snapshot of supplied inputs is taken after formula results are discarded, so an earlier result is never frozen as an input. Comparison. - A missing or crashed base now fails. - A check failing in a year where the base passes is a regression, even if it already failed in another year. - A check the base ran that is missing from the head fails (or is noted, when the partition changed), and the head must have run the core checks. - A figure on one side only is shown as added or removed, never as no change. - A non-finite head figure fails directly. - Households must match one to one. Measurement. - Finiteness is checked after the reform costings. - A rebuilt baseline must match in every year. - A skipped check is recorded as skipped. - metrics_version covers the effective spec, horizon and reforms. - Geographic exclusivity guards are checked on the data. Test data. - Data tests are identified by their marker, so the data-free tests under microsimulation/ run without credentials. - The setup hook runs before pytest's skip handling, so a skip marker cannot turn missing data into a skip. - The latest-data smoke test is a named exception to the pin. The partition's notes now describe main before the #2159 fix correctly. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/data/fiscal_partition.yaml | 20 +- .../test_aggregate_impact_certified.py | 10 + .../test_aggregate_impact_compare.py | 167 +++++++-- .../test_aggregate_impact_data_mode.py | 90 +++++ .../test_aggregate_impact_guard.py | 110 +++++- .../test_aggregate_impact_measure.py | 109 ++++++ policyengine_uk/tests/conftest.py | 31 +- .../utils/aggregate_impact/__main__.py | 7 +- policyengine_uk/utils/aggregate_impact/api.py | 6 +- .../utils/aggregate_impact/compare.py | 134 ++++++-- .../utils/aggregate_impact/guard.py | 322 +++++++++++------- .../utils/aggregate_impact/measure.py | 149 ++++++-- .../utils/aggregate_impact/render.py | 23 +- 13 files changed, 920 insertions(+), 258 deletions(-) create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py create mode 100644 policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py diff --git a/policyengine_uk/data/fiscal_partition.yaml b/policyengine_uk/data/fiscal_partition.yaml index bf03754400..1bd148a09a 100644 --- a/policyengine_uk/data/fiscal_partition.yaml +++ b/policyengine_uk/data/fiscal_partition.yaml @@ -8,9 +8,11 @@ # The heads were specified from the OBR receipts classification and from each # variable's formula, not copied from the model's own lists # (GOV_TAX_VARIABLES, HOUSEHOLD_TAX_VARIABLES, -# PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES). policyengine_uk/tests/ -# test_fiscal_partition.py checks the two against each other and checks the -# accounting properties below on simulated households. +# PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES). +# policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py +# checks the two against each other without data, and the aggregate-impact +# report checks the accounting properties on the certified dataset. The #2159 +# fix adds tests of them on simulated households. # # Spending heads divide gov_spending and household_benefits the same way: # one benefit or service per head, one variable each. @@ -53,7 +55,9 @@ # spending_heads[]: as tax_heads, without obr_line and payer, plus # caseload (default true): whether reports count recipients. # overlap_guards[]: the places double counting could occur, and the -# property the tests check for each. +# property that should hold for each. The aggregate-impact report checks +# the geographic_exclusivity guards on the certified dataset; the #2159 +# fix's tests check the others. schema_version: 1 totals: @@ -437,7 +441,7 @@ tax_heads: exclusions: # Property transaction taxes - - {variable: stamp_duty_land_tax, kind: component, head: sdlt_household, note: "Equal to expected_sdlt, but not zeroed by gov.hmrc.stamp_duty.abolish; report the head, not this."} + - {variable: stamp_duty_land_tax, kind: component, head: sdlt_household, note: "Summed inside expected_sdlt, which until the #2159 fix also adds corporate_sdlt and a stray 0.045; not zeroed by gov.hmrc.stamp_duty.abolish. Report the head, not this."} - {variable: sdlt_on_transactions, kind: component, head: sdlt_household} - {variable: sdlt_on_rent, kind: component, head: sdlt_household} - {variable: sdlt_on_residential_property_transactions, kind: component, head: sdlt_household} @@ -513,7 +517,11 @@ overlap_guards: - id: property_transaction_tax_geography kind: geographic_exclusivity heads: [sdlt_household, land_transaction_tax, land_and_buildings_transaction_tax] - property: At most one is non-zero for any household (England and Northern Ireland, Wales, Scotland). + property: >- + At most one is non-zero for any household (England and Northern Ireland, + Wales, Scotland). Until the #2159 fix this does not hold: expected_sdlt + carries a stray 0.045, and corporate SDLT for shareholders, in every + nation. - id: local_tax_geography kind: geographic_exclusivity heads: [council_tax, domestic_rates] diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py index abce2bb4ce..82ad10326c 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py @@ -186,3 +186,13 @@ def numeric_lists(node): assert longest <= 50 < n_households details = " ".join(d for c in result["checks"] for d in c["details"]) assert "minimum £" not in details and "largest gap" not in details + + +def test_geographic_guards_hold_except_where_a_double_count_is_declared(measured): + """Council tax and domestic rates never meet. SDLT, LTT and LBTT overlap + only while expected_sdlt carries corporate SDLT (#2159).""" + result, _, partition, _ = measured + status = {c["id"]: c["status"] for c in result["checks"]} + assert status["overlap_guard:local_tax_geography"] == "pass" + overlapping = status["overlap_guard:property_transaction_tax_geography"] == "fail" + assert overlapping == bool(partition.double_counts) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py index 1661ac49e0..b408f40cd6 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py @@ -11,6 +11,8 @@ from policyengine_uk.utils.aggregate_impact import COMMENT_MARKER, TOOL_SCHEMA from policyengine_uk.utils.aggregate_impact.compare import ( + REQUIRED_CHECKS, + _match, classify_check, compare, decile_impacts, @@ -32,8 +34,25 @@ finite = st.floats(-1e12, 1e12, allow_nan=False, allow_infinity=False) -def result(values, checks=(), sha=SHA, status="ok", partition_sha="p"): +def check(check_id, status="pass", failing=()): + return { + "id": check_id, + "label": check_id, + "section": "s", + "severity": "fail", + "status": status, + "details": ["d"] if status != "pass" else [], + "failing": { + scope: "d" for scope in (failing or ([""] if status == "fail" else [])) + }, + } + + +def result(values, checks=(), sha=SHA, status="ok", partition_sha="p", required=True): """A measurement with one metric per (section, name) and the given checks.""" + all_checks = [check(c) for c in REQUIRED_CHECKS] if required else [] + for entry in checks: + all_checks.append(check(*entry) if isinstance(entry, tuple) else entry) return { "schema": TOOL_SCHEMA, "status": status, @@ -61,17 +80,7 @@ def result(values, checks=(), sha=SHA, status="ok", partition_sha="p"): } for (section, name), series in values.items() ], - "checks": [ - { - "id": cid, - "label": cid, - "section": "s", - "severity": "fail", - "status": s, - "details": ["d"], - } - for cid, s in checks - ], + "checks": all_checks, "guard": {"status": "skipped", "reason": "test"}, "reforms": [], "timings": {"total": 1}, @@ -95,11 +104,13 @@ def result(values, checks=(), sha=SHA, status="ok", partition_sha="p"): (None, "pass", False, "pass"), ("pass", "fail", False, "regression"), (None, "fail", False, "regression"), + ("skipped", "fail", False, "regression"), ("fail", "fail", False, "pre_existing"), ("fail", "fail", True, "regression"), ("warn", "fail", False, "regression"), ("pass", "warn", False, "new_warning"), ("warn", "warn", False, "warning"), + ("pass", "skipped", False, "skipped"), ("pass", None, False, "removed"), ], ) @@ -107,12 +118,24 @@ def test_check_classification(base, head, blocking, expected): assert classify_check(base, head, blocking) == expected +@settings(max_examples=200, deadline=None) +@given( + base=st.sets(st.sampled_from(["2025", "2026", "2027", ""])), + head=st.sets(st.sampled_from(["2025", "2026", "2027", ""]), min_size=1), +) +def test_a_failure_is_pre_existing_only_if_the_base_fails_everywhere_the_head_does( + base, head +): + verdict = classify_check("fail" if base else "pass", "fail", False, base, head) + assert verdict == ("pre_existing" if base and head <= base else "regression") + + @settings(max_examples=60, deadline=None) @given(values=metric_values) def test_identical_sides_change_nothing(values): side = result(values, checks=[("identity:x", "pass"), ("sign:y", "fail")]) summary = compare(side, copy.deepcopy(side), SPEC) - assert summary["status"] == "pass" + assert summary["status"] == "pass", summary["failures"] assert summary["flags"] == [] assert all(not row["changed"] for row in summary["rows"]) assert {c["classification"] for c in summary["checks"]} == {"pass", "pre_existing"} @@ -150,53 +173,129 @@ def test_flags_follow_the_spec_thresholds(): assert len(compare(result(base), result(large), SPEC)["flags"]) == len(YEARS) +CRASHED = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "error": "Traceback\nValueError: boom", +} +ONE = {("receipts", "a"): [1.0] * len(YEARS)} + + def test_a_crashed_or_missing_head_fails(): - base = result({("receipts", "a"): [1.0] * len(YEARS)}) - crashed = { - "schema": TOOL_SCHEMA, - "status": "crashed", - "error": "Traceback\nValueError: boom", - } - summary = compare(base, crashed, SPEC) + summary = compare(result(ONE), CRASHED, SPEC) assert summary["status"] == "fail" and "boom" in summary["failures"][0] - assert compare(base, None, SPEC)["status"] == "fail" + assert compare(result(ONE), None, SPEC)["status"] == "fail" + + +def test_a_crashed_or_missing_base_fails_unless_it_is_not_required(): + for base in (CRASHED, None): + assert compare(base, result(ONE), SPEC)["status"] == "fail" + relaxed = compare(base, result(ONE), SPEC, require_base=False) + assert relaxed["status"] == "pass" and relaxed["warnings"] def test_without_a_base_the_heads_own_failing_checks_block(): - head = result( - {("receipts", "a"): [1.0] * len(YEARS)}, checks=[("identity:x", "fail")] - ) - summary = compare(None, head, SPEC) + head = result(ONE, checks=[("identity:x", "fail")]) + summary = compare(None, head, SPEC, require_base=False) assert summary["status"] == "fail" assert any("identity:x" in f for f in summary["failures"]) def test_different_datasets_fail(): - values = {("receipts", "a"): [1.0] * len(YEARS)} - summary = compare(result(values), result(values, sha="6" * 64), SPEC) + summary = compare(result(ONE), result(ONE, sha="6" * 64), SPEC) assert summary["status"] == "fail" def test_non_finite_head_figures_block_even_when_the_base_shares_them(): - values = {("receipts", "a"): [1.0] * len(YEARS)} - checks = [("finite:metrics", "fail")] - summary = compare(result(values, checks), result(values, checks), SPEC) + checks = [check("finite:x", "fail")] + summary = compare(result(ONE, checks), result(ONE, checks), SPEC) assert summary["status"] == "fail" +def test_a_non_finite_head_figure_fails_even_if_no_check_caught_it(): + """Reform costings are measured after most checks; compare looks itself.""" + bad = {("receipts", "a"): [1.0] * (len(YEARS) - 1) + [float("nan")]} + summary = compare(result(ONE), result(bad), SPEC) + assert summary["status"] == "fail" + assert any("not finite" in f for f in summary["failures"]) + + +def test_a_newly_failing_year_is_a_regression_not_pre_existing(): + base = result(ONE, checks=[check("identity:x", "fail", ["2025"])]) + same = result(ONE, checks=[check("identity:x", "fail", ["2025"])]) + worse = result(ONE, checks=[check("identity:x", "fail", ["2025", "2026"])]) + assert compare(base, same, SPEC)["status"] == "pass" + summary = compare(base, worse, SPEC) + assert summary["status"] == "fail" + assert any("identity:x" in f for f in summary["failures"]) + + +def test_a_check_that_vanishes_fails_unless_the_partition_changed(): + base = result(ONE, checks=[("sign:vat", "pass")]) + summary = compare(base, result(ONE), SPEC) + assert summary["status"] == "fail" + assert any("sign:vat" in f for f in summary["failures"]) + edited = compare(base, result(ONE, partition_sha="q"), SPEC) + assert edited["status"] == "pass" + assert any("sign:vat" in w for w in edited["warnings"]) + + +def test_a_head_without_the_required_checks_fails(): + summary = compare(result(ONE), result(ONE, required=False), SPEC) + assert summary["status"] == "fail" + assert any("finite:metrics" in f for f in summary["failures"]) + + +def test_a_check_that_no_longer_applies_is_noted_not_failed(): + base = result(ONE, checks=[("counted_once:x", "pass")]) + head = result(ONE, checks=[check("counted_once:x", "skipped")]) + summary = compare(base, head, SPEC) + assert summary["status"] == "pass" + assert any("counted_once:x" in w for w in summary["warnings"]) + + +def test_a_figure_on_one_side_only_is_a_change(): + two = {**ONE, ("receipts", "b"): [2.0] * len(YEARS)} + removed = compare(result(two), result(ONE), SPEC) + row = next(r for r in removed["rows"] if r["id"] == "receipts.b") + assert row["changed"] and row["only_in"] == "base" + text = render(removed, SPEC) + assert "removed: base levels" in text + assert "No change in any year:** receipts" not in text + added = compare(result(ONE), result(two), SPEC) + assert "new: head levels" in render(added, SPEC) + + def test_strict_mode_fails_pre_existing_failures(): - values = {("receipts", "a"): [1.0] * len(YEARS)} - side = result(values, checks=[("sign:vat", "fail")]) + side = result(ONE, checks=[("sign:vat", "fail")]) assert compare(side, copy.deepcopy(side), SPEC)["status"] == "pass" assert compare(side, copy.deepcopy(side), SPEC, strict=True)["status"] == "fail" def test_a_partition_edit_is_noted(): - values = {("receipts", "a"): [1.0] * len(YEARS)} - summary = compare(result(values), result(values, partition_sha="q"), SPEC) + summary = compare(result(ONE), result(ONE, partition_sha="q"), SPEC) assert any("partition" in w for w in summary["warnings"]) +def test_households_match_only_one_to_one(): + assert _match(np.array([1, 2]), np.array([1, 2, 3])) is None + assert _match(np.array([1, 2]), np.array([1, 1])) is None + assert _match(np.array([1, 1]), np.array([1, 1])) is None + assert np.array_equal(_match(np.array([1, 2, 3]), np.array([3, 1, 2])), [1, 2, 0]) + + +@settings(max_examples=100, deadline=None) +@given( + ids=st.lists(st.integers(0, 10_000), min_size=1, max_size=60, unique=True), + data=st.data(), +) +def test_matching_recovers_any_permutation(ids, data): + base = np.array(ids) + head = np.array(data.draw(st.permutations(ids))) + matched = _match(base, head) + assert matched is not None and np.array_equal(head[matched], base) + + def households(n, rng, decile=None, income=None, weight=None): return { str(y): { diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py new file mode 100644 index 0000000000..5268196c19 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py @@ -0,0 +1,90 @@ +"""Mandatory-data mode end to end, in a subprocess with this repo's hooks. + +A scratch test directory imports the real conftest's hooks, so the cases +exercise the hook ordering pytest actually applies. +""" + +import os +import subprocess +import sys +import textwrap + +import pytest + +CONFTEST = "from policyengine_uk.tests.conftest import * # noqa: F401,F403\n" +CASES = textwrap.dedent( + """ + import pytest + + def test_needs_no_data(): + assert True + + @pytest.mark.microsimulation + def test_needs_data(): + assert True + + @pytest.mark.microsimulation + @pytest.mark.skip(reason="a skip marker must not hide missing data") + def test_needs_data_but_is_marked_skip(): + assert True + """ +) + + +def run(tmp_path, **env_changes): + (tmp_path / "conftest.py").write_text(CONFTEST) + (tmp_path / "test_cases.py").write_text(CASES) + env = { + k: v + for k, v in os.environ.items() + if k + not in ( + "HUGGING_FACE_TOKEN", + "POLICYENGINE_UK_DEFAULT_DATASET", + "POLICYENGINE_UK_REQUIRE_TEST_DATA", + "PYTEST_ADDOPTS", + ) + } + env.update(env_changes) + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(tmp_path), + "-q", + "-p", + "no:cacheprovider", + "-p", + "no:xdist", + "--rootdir", + str(tmp_path), + "-W", + "ignore::pytest.PytestUnknownMarkWarning", + ], + env=env, + capture_output=True, + text=True, + timeout=600, + ) + + +def test_without_data_the_data_tests_skip_and_the_rest_run(tmp_path): + done = run(tmp_path) + assert done.returncode == 0, done.stdout + done.stderr + assert "1 passed, 2 skipped" in done.stdout + + +def test_in_mandatory_mode_every_data_test_fails_even_one_marked_skip(tmp_path): + done = run(tmp_path, POLICYENGINE_UK_REQUIRE_TEST_DATA="1") + assert done.returncode != 0 + assert "1 passed, 2 errors" in done.stdout, done.stdout + assert "skipped" not in done.stdout.splitlines()[-1] + assert "Mandatory-data mode" in done.stdout + + +@pytest.mark.parametrize("value", ["maybe", "2"]) +def test_an_unrecognised_mode_value_fails_closed(tmp_path, value): + done = run(tmp_path, POLICYENGINE_UK_REQUIRE_TEST_DATA=value) + assert done.returncode != 0 + assert "2 errors" in done.stdout, done.stdout diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py index 6f98e66ad6..a4cd4cb2b4 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py @@ -80,7 +80,9 @@ def _invalidate_all_caches(self): self.cache = {} -def partition(tax_heads, spending_heads=(), double_counts=()): +def partition(tax_heads, spending_heads=(), double_counts=(), reads=None): + reads = reads or {} + def heads(spec, total): return [ { @@ -89,6 +91,7 @@ def heads(spec, total): "variable": v, "totals": [total], "entity": "household", + "reads": [{"head": r} for r in reads.get(v, [])], } for v in spec ] @@ -151,25 +154,80 @@ def test_stale_double_count_declaration_fails(): assert result["double_counts"][0]["coefficient"] == pytest.approx(0) -def test_undeclared_containment_fails_with_coefficient_two(): +def test_undeclared_containment_fails_because_another_head_moves(): _, result = run(model(contains=True), partition(["tax_a", "tax_b"])) rows = by_variable(result) assert rows["tax_b"]["status"] == "fail" - assert rows["tax_b"]["coefficients"]["gov_tax"] == pytest.approx(2) + assert "tax_a moves by +1.0000" in rows["tax_b"]["detail"] + assert rows["tax_b"]["responses"]["gov_tax"]["tax_a"] == pytest.approx(1) assert rows["tax_a"]["status"] == "pass" def test_means_tested_offset_passes_but_a_benefit_rising_with_income_fails(): _, offset = run(model(interaction=-0.5), partition(["tax_a"], ["ben_c", "ben_d"])) - assert by_variable(offset)["ben_c"]["coefficients"][ - "gov_spending" - ] == pytest.approx(0.5) - assert by_variable(offset)["ben_c"]["status"] == "pass" + row = by_variable(offset)["ben_c"] + assert row["coefficients"]["gov_spending"] == pytest.approx(0.5) + assert row["status"] == "pass" _, rising = run(model(interaction=0.2), partition(["tax_a"], ["ben_c", "ben_d"])) - assert by_variable(rising)["ben_c"]["coefficients"][ - "gov_spending" - ] == pytest.approx(1.2) - assert by_variable(rising)["ben_c"]["status"] == "fail" + row = by_variable(rising)["ben_c"] + assert row["status"] == "fail" and "ben_d rises by 0.2000" in row["detail"] + + +def test_a_total_that_adds_a_zero_component_twice_fails(): + """Baseline identities hold while tax_b is zero; the response gives it away.""" + formulas = model(contains=False) + formulas["tax_b"] = lambda f: 0 * f("x") + formulas["gov_tax"] = lambda f: f("tax_a") + 2 * f("tax_b") + _, result = run(formulas, partition(["tax_a", "tax_b"])) + row = by_variable(result)["tax_b"] + assert row["coefficients"]["gov_tax"] == pytest.approx(1) + assert row["model_coefficients"]["gov_tax"] == pytest.approx(2) + assert ( + row["status"] == "fail" + and "the model's gov_tax moves by 2.0000" in row["detail"] + ) + + +def test_a_means_test_fall_cannot_hide_a_benefit_that_contains_another(): + """ben_e contains ben_c while ben_d falls pound for pound: the net is 1.""" + formulas = model(interaction=-1.0) + formulas["ben_e"] = lambda f: 50 + f("ben_c") + formulas["gov_spending"] = lambda f: f("ben_c") + f("ben_d") + f("ben_e") + _, result = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_e"])) + row = by_variable(result)["ben_c"] + assert row["coefficients"]["gov_spending"] == pytest.approx(1) + assert row["status"] == "fail" and "ben_e rises by 1.0000" in row["detail"] + + +def test_an_eligibility_loss_in_the_same_household_cannot_hide_a_double_count(): + """One household's ben_e contains ben_c and it loses £100 of ben_f at once.""" + formulas = model(interaction=0.0) + first = np.array([1.0, 0.0, 0.0]) + formulas["ben_e"] = lambda f: 50 + first * f("ben_c") + formulas["ben_f"] = lambda f: np.where( + (f("ben_c") > 100.5) & (first == 1), 0.0, 100.0 + ) + formulas["gov_spending"] = lambda f: ( + f("ben_c") + f("ben_d") + f("ben_e") + f("ben_f") + ) + _, result = run( + formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_e", "ben_f"]) + ) + row = by_variable(result)["ben_c"] + assert row["status"] == "fail" and "ben_e rises" in row["detail"] + + +def test_a_declared_read_may_respond_but_not_one_for_one(): + formulas = model(contains=False) + formulas["tax_a"] = lambda f: 0.1 * f("x") + 0.3 * f("tax_b") + reading = partition(["tax_a", "tax_b"], reads={"tax_a": ["tax_b"]}) + _, result = run(formulas, reading) + assert by_variable(result)["tax_b"]["status"] == "pass" + _, undeclared = run(formulas, partition(["tax_a", "tax_b"])) + assert by_variable(undeclared)["tax_b"]["status"] == "fail" + _, containing = run(model(contains=True), reading) + row = by_variable(containing)["tax_b"] + assert row["status"] == "fail" and "contains it" in row["detail"] def test_supplied_inputs_are_restored_after_perturbation(): @@ -184,14 +242,29 @@ def test_supplied_inputs_are_restored_after_perturbation(): assert by_variable(result)["tax_b"]["units"] == 2 -def test_zero_spending_component_is_skipped_but_zero_tax_is_tested(): +def test_zero_components_are_perturbed_everywhere(): formulas = model(contains=False) formulas["tax_b"] = lambda f: 0 * f("x") formulas["ben_c"] = lambda f: 0 * f("x") _, result = run(formulas, partition(["tax_a", "tax_b"], ["ben_c", "ben_d"])) rows = by_variable(result) - assert rows["ben_c"]["status"] == "skipped" assert rows["tax_b"]["status"] == "pass" and rows["tax_b"]["units"] == 3 + assert rows["ben_c"]["status"] == "pass" and rows["ben_c"]["units"] == 3 + + +def test_a_formula_result_computed_earlier_is_not_frozen_as_an_input(): + """On real policyengine-core, a holder's known periods include results.""" + from policyengine_core.country_template import CountryTaxBenefitSystem + from policyengine_core.simulations import SimulationBuilder + + system = CountryTaxBenefitSystem() + sim = SimulationBuilder().build_from_dict( + system, {"persons": {"a": {"salary": {"2026-01": 3000}}}} + ) + sim.calculate("income_tax", "2026-01") + assert sim.get_holder("income_tax").get_known_periods() + snapshot = snapshot_inputs(sim, ["salary", "income_tax"]) + assert set(snapshot) == {"salary"} @settings(max_examples=40, deadline=None) @@ -200,13 +273,18 @@ def test_zero_spending_component_is_skipped_but_zero_tax_is_tested(): weight=st.floats(0.1, 1e4), contains=st.booleans(), ) -def test_coefficient_is_two_exactly_when_a_head_contains_another(x, weight, contains): +def test_the_model_coefficient_is_two_exactly_when_a_head_contains_another( + x, weight, contains +): _, result = run( model(contains=contains), partition(["tax_a", "tax_b"]), x=x, weights=[weight] * len(x), ) - k = by_variable(result)["tax_b"]["coefficients"]["gov_tax"] - assert k == pytest.approx(2 if contains else 1, rel=1e-6) + row = by_variable(result)["tax_b"] + assert row["model_coefficients"]["gov_tax"] == pytest.approx( + 2 if contains else 1, rel=1e-6 + ) + assert (row["status"] == "fail") == contains assert result["reset_reproduces_baseline"] diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py new file mode 100644 index 0000000000..81d3985163 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py @@ -0,0 +1,109 @@ +"""Measurement bookkeeping: check scopes, reform costing checks, digests.""" + +import dataclasses +from types import SimpleNamespace + +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact.measure import ( + Recorder, + _finite_metrics_check, + definitions_digest, + measure_reforms, +) +from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parse_reforms +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +SPEC = load_spec() +YEARS = [2025, 2026] + + +class YearSim: + """A simulation whose gov_balance per household depends on the year.""" + + def __init__(self, balance_by_year): + self.balance = balance_by_year + entity = SimpleNamespace(key="household") + self.tax_benefit_system = SimpleNamespace( + variables={ + "gov_balance": SimpleNamespace(entity=entity), + "household_weight": SimpleNamespace(entity=entity), + } + ) + + def calculate(self, variable, period, map_to=None, unweighted=True): + if variable == "household_weight": + return np.ones(2) + return np.full(2, self.balance[int(period)] / 2) + + +def test_checks_remember_which_years_failed(): + rec = Recorder() + rec.scope = "2025" + rec.check("identity:x", "x", True) + rec.scope = "2026" + rec.check("identity:x", "x", False, "broken") + record = rec.checks["identity:x"] + assert record["status"] == "fail" and record["failing"] == {"2026": "broken"} + + +def test_a_check_that_did_not_apply_is_skipped_not_passed(): + rec = Recorder() + rec.check("counted_once:x", "x", None, "zero for every unit") + assert rec.checks["counted_once:x"]["status"] == "skipped" + rec.check("counted_once:x", "x", False, "broken") + assert rec.checks["counted_once:x"]["status"] == "fail" + + +def test_a_rebuilt_baseline_must_match_in_every_year(): + rec = Recorder() + baseline = {2025: 10.0, 2026: 20.0} + drifting = YearSim({2025: 10.0, 2026: 20.5}) + measure_reforms(lambda reform=None: drifting, None, baseline, {}, [], YEARS, rec) + record = rec.checks["determinism:rebuilt_baseline"] + assert record["status"] == "fail" and list(record["failing"]) == ["2026"] + + rec = Recorder() + measure_reforms( + lambda reform=None: YearSim(baseline), None, baseline, {}, [], YEARS, rec + ) + assert rec.checks["determinism:rebuilt_baseline"]["status"] == "pass" + + +def test_a_non_finite_reform_costing_fails_the_finiteness_check(): + rec = Recorder() + baseline = {2025: 10.0, 2026: 20.0} + reform = parse_reforms({"reforms": [{"name": "Break it", "parameters": {"a": 1}}]}) + + def factory(reform=None): + return YearSim({2025: 11.0, 2026: float("nan")} if reform else baseline) + + measure_reforms(factory, None, baseline, {}, reform, YEARS, rec) + assert rec.checks.get("reform_runs:break_it") is None # recorded by measure() + _finite_metrics_check(rec) + record = rec.checks["finite:metrics"] + assert record["status"] == "fail" and "reforms.break_it" in record["details"][0] + + +def test_the_definitions_digest_follows_what_is_measured(): + reforms = load_reforms() + digest = definitions_digest(SPEC, reforms) + assert digest == definitions_digest(load_spec(), load_reforms()) + shorter = dataclasses.replace(SPEC, last_year=SPEC.last_year - 1) + assert definitions_digest(shorter, reforms) != digest + assert definitions_digest(SPEC, reforms[:-1]) != digest + other_measures = dataclasses.replace(SPEC, raw={**SPEC.raw, "poverty": {}}) + assert definitions_digest(other_measures, reforms) != digest + + +@settings(max_examples=50, deadline=None) +@given(first=st.integers(2020, 2030), length=st.integers(1, 8), drop=st.integers(0, 9)) +def test_the_digest_separates_different_horizons_and_reform_sets(first, length, drop): + reforms = load_reforms() + a = dataclasses.replace(SPEC, first_year=first, last_year=first + length - 1) + b = dataclasses.replace(SPEC, first_year=first, last_year=first + length) + assert definitions_digest(a, reforms) != definitions_digest(b, reforms) + fewer = reforms[:drop] + reforms[drop + 1 :] + assert definitions_digest(a, fewer) != definitions_digest(a, reforms) diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index d116d5f814..5fe86e974c 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -130,16 +130,34 @@ def verify_certified_dataset() -> Optional[str]: return _verification["problem"] +# Test files that read the unpinned latest dataset on purpose, to notice when +# a new data release breaks the model. They are data tests (they need +# credentials, and fail without them in mandatory mode), but what they run on +# is by design not the pinned certified dataset. +UNPINNED_BY_DESIGN = ("test_latest_data_smoke.py",) + + def needs_data(item) -> bool: - # The keyword also matches every test in the microsimulation directory. - return "microsimulation" in item.keywords + """Whether a test carries the microsimulation marker. + + The marker, not the keyword: keywords include directory names, which + would sweep in the data-free tests under ``microsimulation/``. + """ + return item.get_closest_marker("microsimulation") is not None + + +def runs_on_unpinned_data(item) -> bool: + return item.path.name in UNPINNED_BY_DESIGN def pytest_report_header(config): plan = TEST_DATA_PLAN mode = "mandatory" if plan.mandatory else "optional" target = CERTIFIED_DATASET.describe() if plan.pinned else (plan.dataset or "none") - return f"microsimulation data ({mode}): {plan.action} on {target} ({plan.reason})" + return [ + f"microsimulation data ({mode}): {plan.action} on {target} ({plan.reason})", + "unpinned by design: " + ", ".join(UNPINNED_BY_DESIGN), + ] @pytest.fixture(scope="module") @@ -165,11 +183,14 @@ def pytest_collection_modifyitems(config, items): item.add_marker(skip) +@pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item): """Fail data tests in mandatory mode without data; verify pinned bytes. A failure here is reported as an error in the test's setup, so the run - fails rather than passing with the test unexecuted. + fails rather than passing with the test unexecuted. The hook runs before + pytest's own skip handling, so a skip marker on a data test cannot turn + missing data into a skip. """ if not needs_data(item): return @@ -180,7 +201,7 @@ def pytest_runtest_setup(item): "dataset and must not be skipped.", pytrace=False, ) - if plan.action == "run" and plan.pinned: + if plan.action == "run" and plan.pinned and not runs_on_unpinned_data(item): problem = verify_certified_dataset() if problem: pytest.fail(problem, pytrace=False) diff --git a/policyengine_uk/utils/aggregate_impact/__main__.py b/policyengine_uk/utils/aggregate_impact/__main__.py index d3dbf60dee..d0b75a5b8a 100644 --- a/policyengine_uk/utils/aggregate_impact/__main__.py +++ b/policyengine_uk/utils/aggregate_impact/__main__.py @@ -35,15 +35,10 @@ def cmd_measure(args) -> int: "tool": str(Path(__file__).resolve().parent), } try: - from .measure import measure, tool_digest + from .measure import measure from .partition import default_partition_path, load_partition from .spec import DEFAULT_SPEC_PATH, load_spec - provenance["tool_sha256"] = tool_digest() - provenance["metrics_version"] = ( - f"{TOOL_SCHEMA}+{provenance['tool_sha256'][:16]}" - ) - spec_path = Path(args.spec) if args.spec else DEFAULT_SPEC_PATH spec = load_spec(spec_path, years=args.years) provenance["spec_sha256"] = _sha256(spec_path) diff --git a/policyengine_uk/utils/aggregate_impact/api.py b/policyengine_uk/utils/aggregate_impact/api.py index 20445c05aa..5f2ec755fb 100644 --- a/policyengine_uk/utils/aggregate_impact/api.py +++ b/policyengine_uk/utils/aggregate_impact/api.py @@ -3,7 +3,8 @@ ``compute_budget_metrics`` measures one simulation and returns a long-format report: one row per metric and year, plus the identity and partition checks. Two pairs are comparable when their reports share ``metrics_version`` (the -same tool code and spec), whatever model each pair runs; each model's own +same tool code, and the same effective spec, horizon and reforms), whatever +model each pair runs; each model's own fiscal partition is read from the measured package, and its digest is ``partition_version``. @@ -19,7 +20,7 @@ from typing import Callable, Optional, Sequence from . import TOOL_SCHEMA -from .measure import measure, tool_digest +from .measure import measure from .partition import Partition, default_partition_path, load_partition from .reforms import load_reforms from .spec import DEFAULT_SPEC_PATH, Spec, load_spec @@ -137,7 +138,6 @@ def factory(reform=None): if reforms is None: reforms = load_reforms() provenance = dict(provenance or {}) - provenance.setdefault("metrics_version", f"{TOOL_SCHEMA}+{tool_digest()[:16]}") result, _ = measure( sim, factory, diff --git a/policyengine_uk/utils/aggregate_impact/compare.py b/policyengine_uk/utils/aggregate_impact/compare.py index b4052868f1..ec288a4587 100644 --- a/policyengine_uk/utils/aggregate_impact/compare.py +++ b/policyengine_uk/utils/aggregate_impact/compare.py @@ -1,14 +1,17 @@ """Compare a base and a head measurement. -The comparison fails when the head did not run, ran on different data, or -broke a check the base passes; it reports, without failing, checks both sides -fail (pre-existing) and moves above the flag thresholds. +The comparison fails when either side did not run, when they ran on +different data, when the head has a non-finite figure, when a check the base +ran is missing from the head, or when the head fails a check in a scope (a +year) where the base passes it. It reports, without failing, checks that +fail in the same scopes on both sides (pre-existing) and moves above the +flag thresholds. """ from __future__ import annotations import math -from typing import Mapping, Optional +from typing import Iterable, Mapping, Optional import numpy as np @@ -19,6 +22,15 @@ # non-finite or crashed result cannot be compared at all. ALWAYS_BLOCKING_PREFIXES = ("finite:", "weights:") +# Every head measurement must have run these. +REQUIRED_CHECKS = ( + "closure", + "finite:metrics", + "weights:person", + "weights:benunit", + "weights:household", +) + FLAG_KIND_BY_SECTION = { "reforms": "reform_gbp", "reform_targets": "reform_gbp", @@ -46,23 +58,33 @@ def _finite(value) -> bool: def classify_check( - base_status: Optional[str], head_status: Optional[str], blocking: bool + base_status: Optional[str], + head_status: Optional[str], + blocking: bool, + base_failing: Iterable[str] = (), + head_failing: Iterable[str] = (), ) -> str: """How a check's outcome on the head relates to the base. + ``base_failing`` and ``head_failing`` are the scopes (years) each side + failed in. A failure is pre-existing only if the base fails in every + scope the head fails in; a newly failing year is a regression. + Returns one of: ``pass``, ``regression``, ``pre_existing``, ``fixed``, - ``new_warning``, ``warning``, ``removed``. + ``new_warning``, ``warning``, ``skipped``, ``removed``. """ if head_status is None: return "removed" + if head_status == "skipped": + return "skipped" if head_status == "pass": return "fixed" if base_status in ("fail", "warn") else "pass" if head_status == "warn": return "warning" if base_status == "warn" else "new_warning" # head failed - if blocking or base_status in (None, "pass"): + if blocking or base_status != "fail": return "regression" - if base_status == "warn": + if set(head_failing) - set(base_failing): return "regression" return "pre_existing" @@ -81,7 +103,8 @@ def compare_metrics(base: Mapping, head: Mapping, spec: Spec, years) -> tuple: "kind": template["kind"], "unit": template.get("unit"), "values": {}, - "changed": False, + # A figure measured on one side only is a change, never "no change". + "changed": not (head_metric and base_metric), "only_in": None if head_metric and base_metric else ("head" if head_metric else "base"), @@ -115,12 +138,23 @@ def compare_metrics(base: Mapping, head: Mapping, spec: Spec, years) -> tuple: def _match(base_ids: np.ndarray, head_ids: np.ndarray) -> Optional[np.ndarray]: - """Indices into head arrays that line up with base households, or None.""" - if base_ids.shape == head_ids.shape and np.array_equal(base_ids, head_ids): + """Indices into head arrays that line up with base households, or None. + + The two sides must hold exactly the same households, each once. + """ + base_ids, head_ids = np.asarray(base_ids), np.asarray(head_ids) + if base_ids.shape != head_ids.shape: + return None + if np.array_equal(base_ids, head_ids): + if len(np.unique(base_ids)) != len(base_ids): + return None return np.arange(len(head_ids)) + if len(np.unique(base_ids)) != len(base_ids) or len(np.unique(head_ids)) != len( + head_ids + ): + return None order = np.argsort(head_ids, kind="stable") - positions = np.searchsorted(head_ids[order], base_ids) - positions = np.clip(positions, 0, len(order) - 1) + positions = np.clip(np.searchsorted(head_ids[order], base_ids), 0, len(order) - 1) matched = order[positions] if not np.array_equal(head_ids[matched], base_ids): return None @@ -233,7 +267,13 @@ def compare( base_households: Optional[Mapping] = None, head_households: Optional[Mapping] = None, strict: bool = False, + require_base: bool = True, ) -> dict: + """Compare two measurements. + + With ``require_base`` (the default) a missing or crashed base fails the + comparison: without it there is nothing to show the change against. + """ failures, warnings = [], [] if head and head.get("status") == "skipped": return { @@ -262,15 +302,15 @@ def compare( failures.append( "The head measurement crashed: " + _last_line(head.get("error")) ) + base_problem = None if not base: - warnings.append( - "The base measurement produced no result, so nothing is compared against it." - ) + base_problem = "The base measurement produced no result" elif not base_ok: - warnings.append( - "The base measurement crashed, so the head's own checks decide: " - + _last_line(base.get("error")) - ) + base_problem = "The base measurement crashed: " + _last_line(base.get("error")) + if base_problem and require_base: + failures.append(base_problem + "; the change cannot be compared without it.") + elif base_problem: + warnings.append(base_problem + "; the head's own checks decide.") years = (head or base or {}).get("years") or spec.years if head_ok and base_ok: @@ -300,15 +340,48 @@ def compare( runtime_differences = [] base_checks, head_checks = _check_map(base), _check_map(head) + same_partition = ( + base_ok + and head_ok + and (base.get("partition") or {}).get("sha256") + == (head.get("partition") or {}).get("sha256") + ) checks = [] + if head_ok: + missing = [c for c in REQUIRED_CHECKS if c not in head_checks] + if missing: + failures.append( + "The head measurement did not run: " + ", ".join(missing) + "." + ) for check_id in dict.fromkeys(list(head_checks) + list(base_checks)): b, h = base_checks.get(check_id), head_checks.get(check_id) blocking = check_id.startswith(ALWAYS_BLOCKING_PREFIXES) or not base_ok classification = classify_check( - b["status"] if b else None, h["status"] if h else None, blocking + b["status"] if b else None, + h["status"] if h else None, + blocking, + (b or {}).get("failing", {}), + (h or {}).get("failing", {}), ) if strict and classification == "pre_existing": classification = "regression" + if classification == "skipped" and b and b["status"] != "skipped": + warnings.append( + f"The check {check_id} no longer applies on the head: " + + "; ".join((h or {}).get("details", [])[:1]) + ) + if classification == "removed" and head_ok: + # A check can disappear with the head it checked; otherwise its + # disappearance would let a failure pass unseen. + if same_partition: + failures.append( + f"The check {check_id} ran on the base but not on the head." + ) + else: + warnings.append( + f"The check {check_id} ran on the base but not on the head " + "(the partition changed)." + ) template = h or b checks.append( { @@ -329,11 +402,30 @@ def compare( ) rows, flags = compare_metrics(_metric_map(base), _metric_map(head), spec, years) + non_finite = [] for row in rows: if row["only_in"] == "base" and head_ok: warnings.append( f"{row['label']} ({row['id']}) is measured on the base only." ) + elif row["only_in"] == "head" and base_ok: + warnings.append( + f"{row['label']} ({row['id']}) is measured on the head only." + ) + if row["only_in"] != "base": + non_finite += [ + f"{row['id']} {fiscal_year_label(int(year))}" + for year, cell in row["values"].items() + if not _finite(cell["head"]) + ] + if non_finite and head_ok: + more = f" and {len(non_finite) - 10} more" if len(non_finite) > 10 else "" + failures.append( + "The head has figures that are missing or not finite: " + + "; ".join(non_finite[:10]) + + more + + "." + ) deciles, decile_notes, decile_flags = {}, [], [] if head_ok and base_ok and base_households and head_households: diff --git a/policyengine_uk/utils/aggregate_impact/guard.py b/policyengine_uk/utils/aggregate_impact/guard.py index 9850d06f9d..c209c3d53f 100644 --- a/policyengine_uk/utils/aggregate_impact/guard.py +++ b/policyengine_uk/utils/aggregate_impact/guard.py @@ -1,25 +1,34 @@ """The counted-once check: every component enters each total exactly once. For each component of a partition total, the check adds 1 to every unit -where the component is non-zero, recomputes the partition's heads, and -measures how much each total it belongs to moved per unit added (its -coefficient). Heads name one variable each, so a coefficient above 1 means -another head's variable contains the component as well: a double count of -the kind #2159 reports. On tax totals nothing should offset the addition, so -the coefficient must be exactly 1. On spending totals means tests can claw -the addition back (Pension Credit falls when State Pension rises), so the -coefficient may be below 1 but never above it: above 1 means either a double -count or another benefit rising when a household's income rises. Households -whose spending jumps far beyond the addition have crossed an eligibility -threshold; they are left out of the coefficient and counted instead. - -A declared double count (``model_double_counts``) is perturbed too: its -coefficient in its container must still be 1, or the declaration is stale. - -The check reuses one simulation. Between perturbations it discards every -formula result and keeps the supplied inputs, then restores any input it -overrode; at the end it recomputes the baseline and requires it to match the -first computation exactly. +where the component is non-zero, recomputes every head of the totals it +belongs to, and reads each head's response household by household. For a +total T and a component c: + +- c's own head (c itself, or the head a declared double count is contained + in) must rise by exactly 1 per unit added; +- on a tax total, no other head may move at all: a head that moves contains + c, or reads it; +- on a spending total, no other head may rise in any household. Means tests + may lower other benefits when one rises (Pension Credit falls as State + Pension rises), but a benefit that rises with another either contains it + or pays more to people with more income. Rises and falls are not netted, + so a fall elsewhere cannot hide a double count; +- the model's own total must move by exactly what its heads moved, plus one + more per unit if c is a declared double count of T. This catches a total + whose formula adds a component twice even when that component is zero. + +A head that declares a read of c's head (``reads`` in the partition) may +respond, unless it responds one for one, which is containment. A spending +component that is zero for every unit is perturbed everywhere, which makes +recipients of non-recipients; other heads may then rise through passporting, +so only a one-for-one response fails. + +The check reuses one simulation. It takes its snapshot of supplied inputs +only after discarding formula results, so a value some formula computed +earlier is never frozen as an input. Between perturbations it discards every +formula result, and restores any input it overrode; at the end it recomputes +the baseline and requires it to match the first computation exactly. """ from __future__ import annotations @@ -30,12 +39,6 @@ from .partition import Partition -# On spending totals, a household's change counts as an eligibility change -# (and is left out of the coefficient) when it differs from what was added by -# more than this many pounds, or this many times what was added. -ELIGIBILITY_JUMP_GBP = 50.0 -ELIGIBILITY_JUMP_FACTOR = 20.0 - def _raw(sim, variable, period): try: @@ -45,8 +48,23 @@ def _raw(sim, variable, period): return np.asarray(getattr(result, "values", result)) +def discard_formula_results(sim) -> bool: + """Drop every calculated value, keeping supplied inputs. False if unsupported.""" + reset = getattr(sim, "_invalidate_all_caches", None) + if reset is None: + return False + reset() + return True + + def snapshot_inputs(sim, variables) -> dict: - """Copy the supplied input arrays of ``variables`` before any calculation.""" + """Copy the supplied input arrays of ``variables``. + + Formula results are discarded first: a holder's known periods include + calculated values, and restoring one of those as an input would freeze + it for every later perturbation. + """ + discard_formula_results(sim) snapshot = {} for variable in variables: if variable not in sim.tax_benefit_system.variables: @@ -80,17 +98,82 @@ def _household(sim, values, entity): return np.asarray(sim.map_result(values, entity, "household"), dtype=np.float64) -def _totals_hh(sim, partition, year, totals, active=()): - out = {} - for total in totals: - hh = None - for head in partition.heads_in(total, active): - values = _raw(sim, head.variable, year) - entity = sim.tax_benefit_system.variables[head.variable].entity.key - mapped = _household(sim, values, entity) - hh = mapped if hh is None else hh + mapped - out[total] = hh - return out +def _head_arrays(sim, year, variables) -> dict: + """Each variable's values summed to households.""" + definitions = sim.tax_benefit_system.variables + return { + v: _household(sim, _raw(sim, v, year), definitions[v].entity.key) + for v in variables + } + + +def judge_total( + *, + total: str, + is_tax: bool, + component: str, + own_variable: str, + declared_double_count: bool, + zero_baseline: bool, + weights: np.ndarray, + expected: np.ndarray, + head_changes: Mapping[str, np.ndarray], + model_change: Optional[np.ndarray], + allowed_readers: Iterable[str], + tolerance: float, +) -> dict: + """Apply the counted-once rules to one total's responses. + + ``head_changes`` maps each head variable of the total to its change by + household; ``expected`` is what was added to the component by household. + Returns the responses per unit added, the problem with the component's + own head (if it did not rise one for one) and the other problems. + """ + added = float(np.sum(weights * expected)) + allowed = set(allowed_readers) + responses, problems, own_problem = {}, [], None + for variable, change in head_changes.items(): + net = float(np.sum(weights * change)) / added + rise = float(np.sum(weights * np.maximum(change, 0.0))) / added + moved = float(np.sum(weights * np.abs(change))) / added + responses[variable] = net + if variable == own_variable: + off = float(np.sum(weights * np.abs(change - expected))) / added + if off > tolerance: + own_problem = f"{total}: {variable} moves by {net:.4f} per unit of {component}, not 1" + continue + one_for_one = abs(net - 1) <= tolerance and abs(rise - 1) <= tolerance + if variable in allowed or (zero_baseline and not is_tax): + if one_for_one: + problems.append( + f"{total}: {variable} rises one for one with {component}, so it contains it" + ) + elif is_tax: + if moved > tolerance: + problems.append( + f"{total}: {variable} moves by {net:+.4f} per unit of {component}" + ) + elif rise > tolerance: + problems.append( + f"{total}: {variable} rises by {rise:.4f} per unit of {component}" + ) + partition_coefficient = float(sum(responses.values())) + model_coefficient = None + if model_change is not None: + model_coefficient = float(np.sum(weights * model_change)) / added + expected_model = partition_coefficient + (1.0 if declared_double_count else 0.0) + if abs(model_coefficient - expected_model) > tolerance: + problems.append( + f"the model's {total} moves by {model_coefficient:.4f} per unit of " + f"{component}, but its heads account for {expected_model:.4f}" + ) + return { + "responses": responses, + "coefficient": partition_coefficient, + "model_coefficient": model_coefficient, + "own_problem": own_problem, + "problems": problems, + } def counted_once( @@ -103,30 +186,25 @@ def counted_once( variables_to_check: Optional[Iterable[str]] = None, ) -> dict: """Perturb each component (or those in ``variables_to_check``) once.""" - reset = getattr(sim, "_invalidate_all_caches", None) - if reset is None: + if not discard_formula_results(sim): return { "status": "skipped", "reason": "this policyengine-core has no way to discard formula results", } snapshot = snapshot or {} - variables = sim.tax_benefit_system.variables - model_totals = [t for t in partition.totals if t in variables] - partition_totals = [t for t in partition.totals if partition.heads_in(t)] + active = set(active) + definitions = sim.tax_benefit_system.variables + totals = [t for t in partition.totals if partition.heads_in(t, active)] + heads_of = {t: [h.variable for h in partition.heads_in(t, active)] for t in totals} + all_heads = list(dict.fromkeys(v for t in totals for v in heads_of[t])) + model_totals = [t for t in totals if t in definitions] - reset() hw = _raw(sim, "household_weight", year).astype(np.float64) - heads0 = _totals_hh(sim, partition, year, partition_totals, active) + heads0 = _head_arrays(sim, year, all_heads) model0 = {t: _raw(sim, t, year).astype(np.float64) for t in model_totals} - containers = {dc.contained_in for dc in partition.double_counts} - container0 = { - c: _household(sim, _raw(sim, c, year), variables[c].entity.key) - for c in containers - } baseline = {} for variable in partition.component_variables(): - definition = variables[variable] - if definition.definition_period == "month": + if definitions[variable].definition_period == "month": baseline[variable] = [ (month, _raw(sim, variable, month).astype(np.float64)) for month in _months(year) @@ -134,23 +212,22 @@ def counted_once( else: baseline[variable] = _raw(sim, variable, year).astype(np.float64) - results, double_count_results = [], [] declared = {dc.variable: dc for dc in partition.double_counts} + head_by_variable = {h.variable: h for h in partition.heads} selected = None if variables_to_check is None else set(variables_to_check) + results, double_count_results = [], [] for variable in partition.component_variables(): if selected is not None and variable not in selected: continue - definition = variables[variable] + definition = definitions[variable] entity = definition.entity.key - totals = [ - t for t in partition_totals if variable in partition.components(t, active) - ] - tax_totals = [t for t in totals if partition.is_tax_total(t)] + in_totals = [t for t in totals if variable in partition.components(t, active)] record = { "variable": variable, - "totals": totals, + "totals": in_totals, "coefficients": {}, "model_coefficients": {}, + "responses": {}, "units": 0, "status": "pass", "detail": "", @@ -159,87 +236,72 @@ def counted_once( record.update(status="skipped", detail="not a money variable") results.append(record) continue - if definition.definition_period == "month": - year_values = sum(values for _, values in baseline[variable]) - else: - year_values = baseline[variable] + monthly = definition.definition_period == "month" + year_values = ( + sum(values for _, values in baseline[variable]) + if monthly + else baseline[variable] + ) nonzero = year_values != 0 - if nonzero.any(): - mask = nonzero - elif tax_totals: - # A tax that is zero everywhere can still be tested: taxes don't - # passport anyone to anything. - mask = np.ones_like(nonzero) - else: - record.update( - status="skipped", - detail=( - "zero for every unit; adding to it would create recipients, " - "whom passporting rules could treat differently" - ), - ) - results.append(record) - continue + zero_baseline = not bool(nonzero.any()) + mask = np.ones_like(nonzero) if zero_baseline else nonzero added = mask.astype(np.float64) expected = _household(sim, added, entity) - expected_total = float(np.sum(hw * expected)) record["units"] = int(mask.sum()) + own = declared[variable].contained_in if variable in declared else variable + own_head = head_by_variable[own] - reset() - if definition.definition_period == "month": + discard_formula_results(sim) + if monthly: for month, values in baseline[variable]: sim.set_input(variable, month, values + added / 12) else: sim.set_input(variable, year, baseline[variable] + added) - heads1 = _totals_hh(sim, partition, year, totals, active) + needed = list(dict.fromkeys(v for t in in_totals for v in heads_of[t])) + heads1 = _head_arrays(sim, year, needed) problems = [] - record["eligibility_changes"] = {} - for total in totals: - moved = heads1[total] - heads0[total] - if total in tax_totals: - k = float(np.sum(hw * moved)) / expected_total - if abs(k - 1) > tolerance: - problems.append(f"{total} moves by {k:.4f} per unit, not 1") - else: - # A household whose spending jumps by far more than it was - # given has crossed an eligibility threshold rather than - # counted the component twice, which adds exactly one more - # per unit. Leave those households out, and count them. - jumped = np.abs(moved - expected) > np.maximum( - ELIGIBILITY_JUMP_GBP, ELIGIBILITY_JUMP_FACTOR * expected - ) - kept = float(np.sum(hw * expected * ~jumped)) - record["eligibility_changes"][total] = int(jumped.sum()) - if kept <= 0: - k = float("nan") - problems.append( - f"{total}: every perturbed household changed eligibility" - ) - else: - k = float(np.sum(hw * moved * ~jumped)) / kept - if k > 1 + tolerance: - problems.append( - f"{total} moves by {k:.4f} per unit, more than 1" - ) - record["coefficients"][total] = k - for total in model_totals: - if variable in partition.components(total, active): - moved = _raw(sim, total, year).astype(np.float64) - model0[total] - record["model_coefficients"][total] = ( - float(np.sum(hw * moved)) / expected_total - ) + for total in in_totals: + readers = [ + v for v in heads_of[total] if own_head.id in head_by_variable[v].reads + ] + verdict = judge_total( + total=total, + is_tax=partition.is_tax_total(total), + component=variable, + own_variable=own, + declared_double_count=variable in declared + and total in declared[variable].totals, + zero_baseline=zero_baseline, + weights=hw, + expected=expected, + head_changes={v: heads1[v] - heads0[v] for v in heads_of[total]}, + model_change=( + _raw(sim, total, year).astype(np.float64) - model0[total] + if total in model0 + else None + ), + allowed_readers=readers, + tolerance=tolerance, + ) + record["coefficients"][total] = verdict["coefficient"] + if verdict["model_coefficient"] is not None: + record["model_coefficients"][total] = verdict["model_coefficient"] + record["responses"][total] = { + v: r for v, r in verdict["responses"].items() if abs(r) > tolerance + } + problems += verdict["problems"] + # A declared double count whose container no longer moves is + # reported once, as a stale declaration, below. + if verdict["own_problem"] and variable not in declared: + problems.append(verdict["own_problem"]) if variable in declared: dc = declared[variable] - moved = ( - _household( - sim, - _raw(sim, dc.contained_in, year), - variables[dc.contained_in].entity.key, - ) - - container0[dc.contained_in] + container_change = heads1[own] - heads0[own] + off = float(np.sum(hw * np.abs(container_change - expected))) / float( + np.sum(hw * expected) ) - k = float(np.sum(hw * moved)) / expected_total - present = abs(k - 1) <= tolerance + k = float(np.sum(hw * container_change)) / float(np.sum(hw * expected)) + present = off <= tolerance double_count_results.append( { "variable": variable, @@ -261,11 +323,9 @@ def counted_once( results.append(record) _restore(sim, variable, snapshot) - reset() - heads_again = _totals_hh(sim, partition, year, partition_totals, active) - reproduces = all( - np.array_equal(heads_again[t], heads0[t]) for t in partition_totals - ) + discard_formula_results(sim) + heads_again = _head_arrays(sim, year, all_heads) + reproduces = all(np.array_equal(heads_again[v], heads0[v]) for v in all_heads) return { "status": "ran", "year": year, diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py index 73746112fb..f97b390b4a 100644 --- a/policyengine_uk/utils/aggregate_impact/measure.py +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -37,6 +37,9 @@ class Recorder: def __init__(self): self.metrics: dict = {} self.checks: dict = {} + # The year being measured; a check records which scopes it failed + # in, so a comparison can tell a newly failing year from an old one. + self.scope: str = "" def metric( self, metric_id, label, section, kind, year, value, group=None, unit=None @@ -55,9 +58,20 @@ def metric( ) record["values"][str(year)] = float(value) - def check(self, check_id, label, ok, detail="", severity="fail", section=""): - """Record a check; repeated ids (one per year) merge into one result.""" - status = "pass" if ok else ("warn" if severity == "warn" else "fail") + def check( + self, check_id, label, ok, detail="", severity="fail", section="", scope=None + ): + """Record a check; repeated ids (one per year) merge into one result. + + ``failing`` maps each scope the check failed in (a year, or "" for a + check that is not per year) to its detail. ``ok=None`` records that + the check did not apply (``skipped``), which is not a pass. + """ + if ok is None: + status = "skipped" + else: + status = "pass" if ok else ("warn" if severity == "warn" else "fail") + scope = self.scope if scope is None else scope record = self.checks.get(check_id) if record is None: record = { @@ -67,12 +81,20 @@ def check(self, check_id, label, ok, detail="", severity="fail", section=""): "severity": severity, "status": status, "details": [], + "failing": {}, } self.checks[check_id] = record - elif status != "pass" and record["status"] == "pass": + elif status in ("fail", "warn") and record["status"] in ("pass", "skipped"): + record["status"] = status + elif status == "pass" and record["status"] == "skipped": record["status"] = status - if not ok and detail: - record["details"].append(detail) + if ok is None: + if detail and detail not in record["details"]: + record["details"].append(detail) + elif not ok: + record["failing"][scope] = detail + if detail: + record["details"].append(detail) def array(sim, variable: str, period, map_to: Optional[str] = None) -> np.ndarray: @@ -149,6 +171,32 @@ def tool_digest() -> str: return digest.hexdigest() +def definitions_digest(spec: Spec, reforms: Sequence) -> str: + """SHA-256 of what was actually measured: the effective spec and reforms. + + ``tool_digest`` covers the files as shipped; this covers a spec or reform + set passed in instead, and a horizon overridden at the call. + """ + payload = { + "spec": spec.raw, + "years": spec.years, + "guard_year": spec.guard_year, + "reforms": [ + { + "id": r.id, + "parameters": dict(r.parameters), + "scale": dict(r.scale), + "responds": None + if r.responds is None + else [r.responds.variable, r.responds.direction, r.responds.minimum], + } + for r in reforms + ], + } + text = json.dumps(payload, sort_keys=True, default=str) + return hashlib.sha256(text.encode()).hexdigest() + + def runtime_identity() -> dict: """The installed distributions and interpreter that ran the measurement.""" distributions = sorted( @@ -276,6 +324,25 @@ def measure_year( section="coverage", ) + variable_of = {head.id: head.variable for head in partition.heads} + for guard in partition.overlap_guards: + if ( + not isinstance(guard, Mapping) + or guard.get("kind") != "geographic_exclusivity" + ): + continue + members = [variable_of[h] for h in guard.get("heads") or [] if h in variable_of] + nonzero = sum((head_hh[v] != 0).astype(int) for v in members) + overlapping = nonzero > 1 + rec.check( + f"overlap_guard:{guard.get('id')}", + "At most one of " + ", ".join(members) + " is non-zero for any household", + not bool(overlapping.any()), + f"{fiscal_year_label(year)}: {int(overlapping.sum())} households have more " + f"than one, a weighted {float(np.sum(hw * overlapping)):,.0f}", + section="partition", + ) + double_count_hh = {} for dc in partition.double_counts: values = array(sim, dc.variable, year).astype(np.float64) @@ -439,6 +506,12 @@ def partition_sum(total): section="coverage", ) continue + rec.check( + f"available:{measure_spec['id']}", + f"{measure_spec['label']} can be computed", + True, + section="coverage", + ) for group in spec.poverty_groups: if group.get("variable"): try: @@ -452,6 +525,12 @@ def partition_sum(total): section="coverage", ) continue + rec.check( + f"available:{group['id']}", + f"{group['label']} can be identified", + True, + section="coverage", + ) else: members = np.ones_like(in_poverty, dtype=bool) population = float(np.sum(pw * members)) @@ -573,17 +652,18 @@ def measure_reforms( differences that no reform caused. """ outcomes = [] - first = years[0] try: fresh = factory(reform=None) - again = weighted_total(fresh, "gov_balance", first) - rec.check( - "determinism:rebuilt_baseline", - "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", - again == baseline_balance[first], - f"{fiscal_year_label(first)}: £{again:,.0f} against £{baseline_balance[first]:,.0f}", - section="determinism", - ) + for year in years: + again = weighted_total(fresh, "gov_balance", year) + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + again == baseline_balance[year], + f"{fiscal_year_label(year)}: £{again:,.0f} against £{baseline_balance[year]:,.0f}", + section="determinism", + scope=str(year), + ) del fresh except Exception as error: # noqa: BLE001 rec.check( @@ -699,7 +779,9 @@ def measure( reset = getattr(sim, "_invalidate_all_caches", None) for year in spec.years: t = time.time() + rec.scope = str(year) measure_year(sim, partition, spec, year, rec, households) + rec.scope = "" baseline_balance[year] = weighted_total(sim, "gov_balance", year) baseline_targets[year] = { variable: weighted_total(sim, variable, year) @@ -710,7 +792,6 @@ def measure( # Keep memory flat across the horizon; supplied inputs survive. reset() gc.collect() - _finite_metrics_check(rec) guard = {"status": "skipped", "reason": "not requested"} if run_guard: @@ -726,13 +807,11 @@ def measure( ) timings["counted_once"] = round(time.time() - t, 2) for result in guard.get("components", []): - if result["status"] == "skipped": - continue rec.check( f"counted_once:{result['variable']}", - f"Adding £1 to {result['variable']} adds exactly £1 to each tax total it is in, " - "and at most £1 to each spending total", - result["status"] == "pass", + f"Raising {result['variable']} by £1 raises its own head by £1, moves no " + "other tax head and raises no other benefit", + None if result["status"] == "skipped" else result["status"] == "pass", result["detail"], section="counted_once", ) @@ -752,15 +831,14 @@ def measure( "recomputed baseline totals differ from the first run", section="counted_once", ) - if guard.get("status") == "skipped": - rec.check( - "counted_once:ran", - "The counted-once check ran", - False, - guard.get("reason", ""), - severity="warn", - section="counted_once", - ) + rec.check( + "counted_once:ran", + "The counted-once check ran", + guard.get("status") == "ran", + guard.get("reason", ""), + severity="warn", + section="counted_once", + ) t = time.time() reform_outcomes = measure_reforms( @@ -781,14 +859,23 @@ def measure( outcome["error"] or "", section="reforms", ) + # After the reform costings, so that they are covered too. + _finite_metrics_check(rec) timings["total"] = round(time.time() - started, 2) + provenance = dict(provenance or {}) + provenance["tool_sha256"] = tool_digest() + provenance["definitions_sha256"] = definitions_digest(spec, reforms) + provenance["metrics_version"] = ( + f"{TOOL_SCHEMA}+{provenance['tool_sha256'][:16]}" + f"+{provenance['definitions_sha256'][:16]}" + ) result = { "schema": TOOL_SCHEMA, "status": "ok", "label": label, "years": spec.years, - "provenance": dict(provenance or {}), + "provenance": provenance, "runtime": runtime_identity(), "partition": partition.as_dict(), "metrics": list(rec.metrics.values()), diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py index 8924e15a2b..00830fd275 100644 --- a/policyengine_uk/utils/aggregate_impact/render.py +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -92,11 +92,24 @@ def _changes_table(rows, years) -> str: return "No change in any year." body = [] for row in changed: - label = row["label"] if not row.get("group") else f"{row['label']}" - body.append( - [label] - + [fmt_change(row["values"][str(y)]["delta"], row["kind"]) for y in years] - ) + if row["only_in"] == "head": + cells = [ + fmt_level(row["values"][str(y)]["head"], row["kind"]) for y in years + ] + body.append([f"{row['label']} (new: head levels)"] + cells) + elif row["only_in"] == "base": + cells = [ + fmt_level(row["values"][str(y)]["base"], row["kind"]) for y in years + ] + body.append([f"{row['label']} (removed: base levels)"] + cells) + else: + body.append( + [row["label"]] + + [ + fmt_change(row["values"][str(y)]["delta"], row["kind"]) + for y in years + ] + ) return _table(["Measure"] + _year_header(years), body) From bd8bff74c0baea19c09eedf9ce1391d598d2ffaf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:49:52 -0400 Subject: [PATCH 09/10] Carry each check's failing years in the long-format report Qualification compares a candidate's report with the previous pair's. With only a status per check, a check that fails in more years than before looks unchanged; failing_scopes lets it be told apart. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .../test_aggregate_impact_measure.py | 35 +++++++++++++++++++ policyengine_uk/utils/aggregate_impact/api.py | 3 ++ 2 files changed, 38 insertions(+) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py index 81d3985163..94fc65a0e2 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py @@ -7,6 +7,7 @@ from hypothesis import given, settings from hypothesis import strategies as st +from policyengine_uk.utils.aggregate_impact.api import to_report from policyengine_uk.utils.aggregate_impact.measure import ( Recorder, _finite_metrics_check, @@ -107,3 +108,37 @@ def test_the_digest_separates_different_horizons_and_reform_sets(first, length, assert definitions_digest(a, reforms) != definitions_digest(b, reforms) fewer = reforms[:drop] + reforms[drop + 1 :] assert definitions_digest(a, fewer) != definitions_digest(a, reforms) + + +def test_the_long_format_report_carries_each_checks_failing_years(): + rec = Recorder() + rec.scope = "2025" + rec.check("identity:x", "x", False, "broken in 2025") + rec.scope = "2026" + rec.check("identity:x", "x", False, "broken in 2026") + rec.scope = "" + rec.check("closure", "closure", True) + rec.check("counted_once:y", "y", None, "zero for every unit") + rec.metric("receipts.income_tax", "Income tax", "receipts", "gbp", 2025, 1.0) + report = to_report( + { + "status": "ok", + "years": [2025, 2026], + "provenance": {"metrics_version": "v"}, + "partition": {"sha256": "p"}, + "metrics": list(rec.metrics.values()), + "checks": list(rec.checks.values()), + } + ) + rows = {row["identity_id"]: row for row in report["identities"]} + assert rows["identity:x"]["failing_scopes"] == ["2025", "2026"] + assert rows["identity:x"]["status"] == "fail" + assert rows["closure"]["failing_scopes"] == [] + assert rows["counted_once:y"]["status"] == "skipped" + assert rows["counted_once:y"]["failing_scopes"] == [] + metric = report["metrics"][0] + assert ( + metric["metric_id"] == "receipts.income_tax" + and metric["partition"] == "income_tax" + ) + assert metric["kind"] == "receipt" and metric["year"] == 2025 diff --git a/policyengine_uk/utils/aggregate_impact/api.py b/policyengine_uk/utils/aggregate_impact/api.py index 5f2ec755fb..a22eeb11d2 100644 --- a/policyengine_uk/utils/aggregate_impact/api.py +++ b/policyengine_uk/utils/aggregate_impact/api.py @@ -93,6 +93,9 @@ def to_report(result: dict) -> dict: "year": None, "status": check["status"], "severity": check["severity"], + # The years the check failed in ("" for a check with no year), + # so a consumer can tell a newly failing year from an old one. + "failing_scopes": sorted(check.get("failing") or {}), "detail": "; ".join(check["details"]), } ) From 9b33fe19f80676fc5b81d2d80b36072e1c2fc1df Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:55:50 -0400 Subject: [PATCH 10/10] Stamp a measurement with the tool code that was loaded The tool digest was read from disk when a measurement finished, so editing the tool during a run stamped the result with code that had not produced it. It is now taken when the module loads. Refs #2334. Co-Authored-By: Claude Opus 5.5 --- .../test_aggregate_impact_measure.py | 7 +++++++ policyengine_uk/utils/aggregate_impact/measure.py | 13 +++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py index 94fc65a0e2..2bbafc779a 100644 --- a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py @@ -11,8 +11,10 @@ from policyengine_uk.utils.aggregate_impact.measure import ( Recorder, _finite_metrics_check, + _digest_tool_files, definitions_digest, measure_reforms, + tool_digest, ) from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parse_reforms from policyengine_uk.utils.aggregate_impact.spec import load_spec @@ -142,3 +144,8 @@ def test_the_long_format_report_carries_each_checks_failing_years(): and metric["partition"] == "income_tax" ) assert metric["kind"] == "receipt" and metric["year"] == 2025 + + +def test_the_tool_digest_is_taken_at_load_and_matches_the_files(): + assert tool_digest() == _digest_tool_files() + assert len(tool_digest()) == 64 diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py index f97b390b4a..7b0dff4d98 100644 --- a/policyengine_uk/utils/aggregate_impact/measure.py +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -160,8 +160,7 @@ def slug(text: str) -> str: return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") -def tool_digest() -> str: - """SHA-256 of this package's code and spec, independent of where it lives.""" +def _digest_tool_files() -> str: root = Path(__file__).resolve().parent digest = hashlib.sha256() for path in sorted(root.rglob("*")): @@ -171,6 +170,16 @@ def tool_digest() -> str: return digest.hexdigest() +# Taken when this module loads, so a result is stamped with the code that +# produced it even if the files change while a measurement runs. +_TOOL_DIGEST = _digest_tool_files() + + +def tool_digest() -> str: + """SHA-256 of this package's code and spec, independent of where it lives.""" + return _TOOL_DIGEST + + def definitions_digest(spec: Spec, reforms: Sequence) -> str: """SHA-256 of what was actually measured: the effective spec and reforms.