diff --git a/.github/workflows/aggregate-impact.yaml b/.github/workflows/aggregate-impact.yaml new file mode 100644 index 0000000000..b3679d74dd --- /dev/null +++ b/.github/workflows/aggregate-impact.yaml @@ -0,0 +1,204 @@ +# Measure a pull request's base and head on the certified dataset and post +# the comparison: receipts by tax head, spending and caseloads, poverty, +# inequality, deciles and the standard reform costings, 2025-26 to 2031-32. +# +# Both sides run the pull request head's copy of the measurement tool +# (policyengine_uk/utils/aggregate_impact), each in an environment synced +# from its own uv.lock, on the dataset pinned in +# policyengine_uk/tests/certified_dataset.yaml (bytes verified on each side). +# Each side reads its own fiscal partition. The job fails on a crash, a +# different dataset, non-finite figures or a check the head breaks that the +# base passes; moves above the flag thresholds are listed, not failed. +# +# DATA PROTECTION. The dataset is licensed survey microdata. Only aggregates +# may leave this job: the two result.json files, the comment and the summary. +# Never upload the measurement's private/ directory (household-level detail +# for the decile comparison), and never put the dataset in the Actions cache. +# Both sides are measured in one job so that detail stays on the runner. +# test_aggregate_impact_workflow.py enforces this. + +name: Aggregate impact + +on: + pull_request: + branches: + - main + paths: + - policyengine_uk/** + - pyproject.toml + - uv.lock + - .github/workflows/aggregate-impact.yaml + workflow_dispatch: + inputs: + base: + description: Base commit, branch or tag + default: main + head: + description: Head commit, branch or tag (default the dispatched ref) + default: "" + +concurrency: + group: aggregate-impact-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + UV_PYTHON: "3.13" + PYTHONUNBUFFERED: "1" + +jobs: + report: + name: Aggregate impact + # Fork pull requests cannot read the private certified dataset. + if: >- + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04-arm + timeout-minutes: 60 + permissions: + contents: read + issues: write + pull-requests: write + steps: + - name: Choose the commits + id: commits + env: + EVENT: ${{ github.event_name }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + PR_BASE_REF: ${{ github.event.pull_request.base.ref }} + PR_NUMBER: ${{ github.event.pull_request.number }} + INPUT_BASE: ${{ inputs.base }} + INPUT_HEAD: ${{ inputs.head }} + run: | + # A pull request's head is its merge commit, whose first parent is + # the base, so the comparison is exactly what merging would change. + if [ "$EVENT" = "pull_request" ]; then + base="$PR_BASE"; head="$GITHUB_SHA" + base_name="$PR_BASE_REF"; head_name="#$PR_NUMBER merged into $PR_BASE_REF" + else + base="${INPUT_BASE:-main}"; head="${INPUT_HEAD:-$GITHUB_SHA}" + base_name="$base"; head_name="${INPUT_HEAD:-$GITHUB_REF_NAME}" + fi + { + echo "base=$base" + echo "head=$head" + echo "base_name=$base_name" + echo "head_name=$head_name" + } >> "$GITHUB_OUTPUT" + + - name: Check out the head + uses: actions/checkout@v6 + with: + ref: ${{ steps.commits.outputs.head }} + path: head + + - name: Check out the base + uses: actions/checkout@v6 + with: + ref: ${{ steps.commits.outputs.base }} + path: base + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Sync each side's environment from its own lock + run: | + (cd base && uv sync --frozen --no-dev) + (cd head && uv sync --frozen --no-dev) + + - name: Measure base, then head + id: measure + env: + HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + ACTOR: ${{ github.actor }} + BASE_NAME: ${{ steps.commits.outputs.base_name }} + HEAD_NAME: ${{ steps.commits.outputs.head_name }} + run: | + mkdir -p measured/base measured/head + if [ -z "$HUGGING_FACE_TOKEN" ]; then + if [ "$ACTOR" = "dependabot[bot]" ]; then + echo "::notice::Dependabot runs have no data access; the aggregate impact report is skipped." + echo '{"schema": "policyengine-uk-aggregate-impact/1", "status": "skipped", "reason": "Dependabot runs have no access to the certified dataset"}' \ + > measured/head/result.json + exit 0 + fi + echo "::error::HUGGING_FACE_TOKEN is not available, so the certified dataset cannot be read." + exit 1 + fi + tool="$RUNNER_TEMP/tool" + mkdir -p "$tool" + cp -R head/policyengine_uk/utils/aggregate_impact "$tool/pe_uk_aggregate_impact" + measure() { + side="$1"; name="$2" + # A crash is recorded in result.json for the comparison to judge. + (cd "$side" && PYTHONPATH="$tool" .venv/bin/python -m pe_uk_aggregate_impact measure \ + --label "$side" \ + --pin "$GITHUB_WORKSPACE/head/policyengine_uk/tests/certified_dataset.yaml" \ + --fallback-partition "$GITHUB_WORKSPACE/head/policyengine_uk/data/fiscal_partition.yaml" \ + --git-sha "$(git rev-parse HEAD)" \ + --git-ref "$name" \ + --out "$GITHUB_WORKSPACE/measured/$side") || true + } + measure base "$BASE_NAME" + measure head "$HEAD_NAME" + if [ ! -f measured/head/result.json ]; then + echo "::error::The head measurement wrote no result." + exit 1 + fi + + - name: Compare base with head + id: compare + run: | + mkdir -p report/base report/head + base_args=() + if [ -f measured/base/result.json ]; then + base_args=(--base measured/base) + cp measured/base/result.json report/base/result.json + fi + cp measured/head/result.json report/head/result.json + set +e + PYTHONPATH="$RUNNER_TEMP/tool" head/.venv/bin/python -m pe_uk_aggregate_impact compare \ + "${base_args[@]}" \ + --head measured/head \ + --markdown report/comment.md \ + --summary report/summary.json \ + --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + echo "status=$?" >> "$GITHUB_OUTPUT" + set -e + cat report/comment.md >> "$GITHUB_STEP_SUMMARY" + # Household-level detail is no longer needed; remove it before any + # later step could pick it up. + rm -rf measured + + - name: Post or update the pull request comment + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + run: | + marker='' + existing=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \ + --jq ".[] | select(.user.login == \"github-actions[bot]\") | select(.body | startswith(\"$marker\")) | .id" \ + | tail -n 1) + if [ -n "$existing" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$existing" -F body=@report/comment.md > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@report/comment.md > /dev/null + fi + + - name: Upload the report (aggregates only) + if: always() + uses: actions/upload-artifact@v4 + with: + name: aggregate-impact-report + path: report + if-no-files-found: ignore + retention-days: 30 + + - name: Fail on a structural failure + if: steps.compare.outputs.status != '0' + run: | + echo "::error::The aggregate impact report found failures; see the job summary." + exit 1 diff --git a/changelog.d/2334-test-data.changed.md b/changelog.d/2334-test-data.changed.md new file mode 100644 index 0000000000..c82ac32d4d --- /dev/null +++ b/changelog.d/2334-test-data.changed.md @@ -0,0 +1 @@ +- Microsimulation tests now run on the certified uk-data 1.58.0 dataset, pinned by commit and SHA-256, and fail instead of skipping when `POLICYENGINE_UK_REQUIRE_TEST_DATA` is set and the data is unavailable; the reform-cost fixtures are rebased onto it, keeping their earlier values as receipts. diff --git a/changelog.d/2334.added.md b/changelog.d/2334.added.md new file mode 100644 index 0000000000..c58137024c --- /dev/null +++ b/changelog.d/2334.added.md @@ -0,0 +1 @@ +- Added an aggregate-impact report (`python -m policyengine_uk.utils.aggregate_impact`) that measures a model checkout on the pinned certified dataset and compares base with head: receipts by tax head, spending and caseloads, poverty, inequality, deciles and the standard reform costings over 2025-26 to 2031-32, with accounting identities and a counted-once check against a reviewed fiscal partition (`policyengine_uk/data/fiscal_partition.yaml`). diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 66cb54d3c4..9657daece1 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -25,6 +25,22 @@ Use this skill whenever adding, moving, or reviewing tests. private H5 files in unit-style tests. Mock those seams or skip cleanly when the external artifact is unavailable. +## Microsimulation Data + +Microsimulation tests run on one certified dataset, pinned by Hugging Face +commit and SHA-256 in `policyengine_uk/tests/certified_dataset.yaml`. With +`HUGGING_FACE_TOKEN` set, `conftest.py` selects that dataset and verifies its +bytes before the first data test runs. Without a token, data tests skip. + +Set `POLICYENGINE_UK_REQUIRE_TEST_DATA=1` in jobs that must run the data tests. +Each data test then fails at setup when the dataset is unavailable, instead of +skipping, and runs only on the pinned dataset. Credential-free tests are +unaffected. + +The reform fixtures in `microsimulation/reforms_config.yaml` record the dataset +they were computed on. When the pin moves, recompute them on the new dataset, +and add each new value under `receipts` rather than overwriting the old ones. + ## Commands Run lint and formatting before committing: diff --git a/policyengine_uk/data/fiscal_partition.yaml b/policyengine_uk/data/fiscal_partition.yaml new file mode 100644 index 0000000000..1bd148a09a --- /dev/null +++ b/policyengine_uk/data/fiscal_partition.yaml @@ -0,0 +1,959 @@ +# Fiscal partition: how the model's tax and spending totals divide into heads. +# +# Each head is one receipt (an OBR current-receipts line, split by who pays +# where the model splits it) and names the one variable that measures the +# household's share of it. Heads are mutually exclusive: no head's value +# contains another head's value, and no variable is summed twice in a total. +# +# The heads were specified from the OBR receipts classification and from each +# variable's formula, not copied from the model's own lists +# (GOV_TAX_VARIABLES, HOUSEHOLD_TAX_VARIABLES, +# PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES). +# policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py +# checks the two against each other without data, and the aggregate-impact +# report checks the accounting properties on the certified dataset. The #2159 +# fix adds tests of them on simulated households. +# +# Spending heads divide gov_spending and household_benefits the same way: +# one benefit or service per head, one variable each. +# +# This version describes main before the #2159 fix: expected_sdlt still adds +# corporate_sdlt, so the household SDLT head carries corporate SDLT too and +# corporate_sdlt is declared under model_double_counts. The fix gives +# corporate_sdlt its own head (sdlt_corporate) and empties that section. +# +# Consumers (tests, the per-PR aggregate impact report) read this file +# directly. Bump schema_version on any change to field meanings. +# +# Fields +# tax_heads[]: +# id one per head +# label sentence-case name +# variable the one model variable that measures the head +# entity household or person +# obr_line OBR current-receipts line the head belongs to, or +# not_an_obr_current_receipt +# report_group grouping for reports +# payer who bears it in the model: household (the household +# or its members pay it directly), employer, consumer +# incidence (allocated by consumption), corporate +# incidence (allocated by shareholding), or mixed +# totals the totals that include it +# sign nonnegative (at current-law parameter values, for +# non-negative inputs) or any (a change measure) +# zero_at_baseline expected: current-law parameters make it zero for +# every household (a reform-only head) +# reads other heads this head's formula reaches, with the +# route; reads that change eligibility, not inclusion +# exclusions[]: variables that must never be summed alongside the heads. +# kind: component (summed or read inside `head`), measure_within (an +# analytical decomposition already inside `head` through its inputs), +# composite (spans several heads), duplicate (same value as `head`), +# baseline_measure. +# model_double_counts[]: known violations still in the model: a variable a +# total lists although the head variable `contained_in` already adds it. +# spending_heads[]: as tax_heads, without obr_line and payer, plus +# caseload (default true): whether reports count recipients. +# overlap_guards[]: the places double counting could occur, and the +# property that should hold for each. The aggregate-impact report checks +# the geographic_exclusivity guards on the certified dataset; the #2159 +# fix's tests check the others. +schema_version: 1 + +totals: + gov_tax: + label: Government tax revenue in respect of the household + description: Receipts at their level, including employer-paid taxes and VAT. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + - high_value_council_tax_surcharge + household_tax: + label: Taxes borne by the household + description: >- + Taxes subtracted from household net income. VAT and employer NI enter + only as reform-induced changes (vat_change and the employer NI response + incidence), because baseline VAT and employer NI are not deducted from + the incomes the model starts from. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + - high_value_council_tax_surcharge + pre_budget_change_household_tax: + label: Taxes borne by the household, pre-budget-change definition + description: >- + A fixed earlier definition of household_tax used by + pre_budget_change_household_net_income. It omits heads added since + (alcohol, tobacco and vehicle excise duties, private school VAT, the + high value council tax surcharge, student loan repayments, and the + incidence-change heads). + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax + + gov_spending: + label: Government spending in respect of the household + description: >- + Cash benefits, childcare and student support, and public services in + kind (NHS, education, transport subsidies). + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax_benefit + household_benefits: + label: Benefits received by the household + description: >- + The benefit side of household net income. It omits pawhp and the + contributed budget-change line, and adds Scottish Child Payment, Healthy + Start and the Two Child Limit Payment, which gov_spending leaves out. + dropped_when: + gov.contrib.abolish_council_tax: + - council_tax_benefit + +tax_heads: + # Income and earnings + - id: income_tax + label: Income tax + variable: income_tax + entity: person + obr_line: Income tax + report_group: income_tax + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: nics_individual + label: National Insurance (employee and self-employed) + variable: national_insurance + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: nics_employer + label: National Insurance (employer) + variable: ni_employer + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: employer + totals: [gov_tax] + sign: nonnegative + - id: employer_nics_response_capital + label: Employer NI change borne by capital + variable: employer_ni_response_capital_incidence + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: corporate incidence + totals: [household_tax] + sign: any + zero_at_baseline: expected + - id: employer_nics_response_consumer + label: Employer NI change borne by consumers + variable: employer_ni_response_consumer_incidence + entity: person + obr_line: National insurance contributions + report_group: national_insurance + payer: consumer incidence + totals: [household_tax] + sign: any + zero_at_baseline: expected + - id: capital_gains_tax + label: Capital gains tax + variable: capital_gains_tax + entity: person + obr_line: Capital gains tax + report_group: capital_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: student_loan_repayments + label: Student loan repayments + variable: student_loan_repayments + entity: person + obr_line: not_an_obr_current_receipt + obr_line_note: >- + A financial transaction: repayments reduce the student loan book, not + borrowing, so they are outside current receipts. The model counts them + in gov_tax and household_tax; reports should show them separately. + report_group: financial_transactions + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + + # Consumption + - id: vat + label: VAT + variable: vat + entity: household + obr_line: Value added tax + report_group: vat + payer: consumer incidence + totals: [gov_tax] + sign: nonnegative + - id: vat_change + label: Change in VAT from its baseline + variable: vat_change + entity: household + obr_line: Value added tax + report_group: vat + payer: consumer incidence + totals: [household_tax, pre_budget_change_household_tax] + sign: any + zero_at_baseline: expected + reads: + - head: vat + route: vat_change adds vat and subtracts baseline_vat + why: a change measure built from the level; the two are never in the same total + - id: private_school_vat + label: VAT on private school fees + variable: private_school_vat + entity: household + obr_line: Value added tax + report_group: vat + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + zero_at_baseline: expected + zero_at_baseline_note: >- + gov.contrib.labour.private_school_vat is 0 at current law in this + model, so the head is reform-only here. + reads: + - head: income_tax + route: private_school_vat -> attends_private_school -> income_tax + why: attendance is imputed from income; it decides who attends, not how much tax is counted + - head: nics_individual + route: private_school_vat -> attends_private_school -> (net income) -> national_insurance + why: as above + - head: council_tax + route: private_school_vat -> attends_private_school -> household_benefits -> council_tax_benefit -> council_tax + why: as above + - id: fuel_duty + label: Fuel duties + variable: fuel_duty + entity: household + obr_line: Fuel duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: alcohol_duty + label: Alcohol duties + variable: alcohol_duty + entity: household + obr_line: Alcohol duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: tobacco_duty + label: Tobacco duties + variable: tobacco_duty + entity: household + obr_line: Tobacco duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: vehicle_excise_duty + label: Vehicle excise duties + variable: vehicle_excise_duty + entity: household + obr_line: Vehicle excise duties + report_group: excise_duties + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + data_note: >- + Zero for every household on policyengine-uk-data 1.58.0 + (enhanced_frs_2024_25) in every year from 2025 to 2030, measured + 2026-10-11. + - id: tv_licence + label: TV licence fee + variable: tv_licence + entity: household + obr_line: Licence fee receipts + report_group: other_receipts + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + reads: + - head: income_tax + route: tv_licence -> tv_licence_discount -> pension_credit -> pension_credit_income -> income_tax + why: the over-75 discount depends on Pension Credit receipt, a means test on net income + - head: nics_individual + route: tv_licence -> tv_licence_discount -> pension_credit -> pension_credit_income -> national_insurance + why: as above + + # Property + - id: council_tax + label: Council tax + variable: council_tax + entity: household + obr_line: Council tax + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: high_value_council_tax_surcharge + label: High value council tax surcharge + variable: high_value_council_tax_surcharge + entity: household + obr_line: Council tax + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax] + sign: nonnegative + - id: domestic_rates + label: Northern Ireland domestic rates + variable: domestic_rates + entity: household + obr_line: Council tax + obr_line_note: >- + Northern Ireland's domestic rates are its counterpart to council tax; + check the line against the OBR table in use before publishing a + line-by-line comparison. + report_group: local_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: business_rates + label: Business rates + variable: business_rates + entity: household + obr_line: Business rates + report_group: business_rates + payer: corporate incidence + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + - id: sdlt_household + label: Stamp duty land tax (own purchases, and corporate until the #2159 fix) + variable: expected_sdlt + entity: household + obr_line: Stamp duty land tax + report_group: property_transaction_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + Before the #2159 fix expected_sdlt adds corporate_sdlt and a stray 0.045 + per household, so this head is both parts of the receipt. + - id: land_transaction_tax + label: Land transaction tax (Wales) + variable: expected_ltt + entity: household + obr_line: Land transaction tax + report_group: property_transaction_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + expected_ltt multiplies an already purchase-drawn liability by + property_sale_rate, discounting it twice (issue 2246). + - id: land_and_buildings_transaction_tax + label: Land and buildings transaction tax (Scotland) + variable: expected_lbtt + entity: household + obr_line: Land and buildings transaction tax + report_group: property_transaction_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + data_note: >- + expected_lbtt multiplies an already purchase-drawn liability by + property_sale_rate, discounting it twice (issue 2246). + + # Reform-only heads + - id: corporate_incident_tax_change + label: Unspecified tax change borne by capital + variable: corporate_incident_tax_revenue_change + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A lump-sum revenue change allocated by shareholding. + report_group: incidence_changes + payer: corporate incidence + totals: [gov_tax, household_tax] + sign: any + zero_at_baseline: expected + - id: consumer_incident_tax_change + label: Unspecified tax change borne by consumers + variable: consumer_incident_tax_revenue_change + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A lump-sum revenue change allocated by consumption. + report_group: incidence_changes + payer: consumer incidence + totals: [gov_tax, household_tax] + sign: any + zero_at_baseline: expected + - id: wealth_tax + label: Wealth tax + variable: wealth_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: non_primary_residence_wealth_tax + label: Wealth tax excluding the main residence + variable: non_primary_residence_wealth_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: household + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: land_value_tax + label: Land value tax + variable: LVT + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + - id: carbon_tax + label: Carbon tax + variable: carbon_tax + entity: household + obr_line: not_an_obr_current_receipt + obr_line_note: A hypothetical tax. + report_group: hypothetical_taxes + payer: mixed + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + sign: nonnegative + zero_at_baseline: expected + +exclusions: + # Property transaction taxes + - {variable: stamp_duty_land_tax, kind: component, head: sdlt_household, note: "Summed inside expected_sdlt, which until the #2159 fix also adds corporate_sdlt and a stray 0.045; not zeroed by gov.hmrc.stamp_duty.abolish. Report the head, not this."} + - {variable: sdlt_on_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_rent, kind: component, head: sdlt_household} + - {variable: sdlt_on_residential_property_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_non_residential_property_transactions, kind: component, head: sdlt_household} + - {variable: sdlt_on_residential_property_rent, kind: component, head: sdlt_household} + - {variable: sdlt_on_non_residential_property_rent, kind: component, head: sdlt_household} + - {variable: land_transaction_tax, kind: component, head: land_transaction_tax, note: "The liability before the property_sale_rate scaling (issue 2246)."} + - {variable: ltt_on_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_rent, kind: component, head: land_transaction_tax} + - {variable: ltt_on_residential_property_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_non_residential_property_transactions, kind: component, head: land_transaction_tax} + - {variable: ltt_on_residential_property_rent, kind: component, head: land_transaction_tax} + - {variable: ltt_on_non_residential_property_rent, kind: component, head: land_transaction_tax} + - {variable: land_and_buildings_transaction_tax, kind: component, head: land_and_buildings_transaction_tax, note: "The liability before the property_sale_rate scaling (issue 2246)."} + - {variable: lbtt_on_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_residential_property_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_non_residential_property_transactions, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_residential_property_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: lbtt_on_non_residential_property_rent, kind: component, head: land_and_buildings_transaction_tax} + - {variable: corporate_tax_incidence, kind: composite, spans: [sdlt_household, business_rates]} + - {variable: hbai_excluded_income, kind: composite, spans: [sdlt_household, business_rates], note: "Minus corporate_tax_incidence."} + - {variable: baseline_business_rates, kind: duplicate, head: business_rates, note: "Same formula and current-law parameters as business_rates."} + # Income tax and National Insurance + - {variable: income_tax_pre_charges, kind: component, head: income_tax} + - {variable: earned_income_tax, kind: component, head: income_tax} + - {variable: savings_income_tax, kind: component, head: income_tax} + - {variable: dividend_income_tax, kind: component, head: income_tax} + - {variable: property_income_tax, kind: component, head: income_tax} + - {variable: CB_HITC, kind: component, head: income_tax} + - {variable: personal_pension_contributions_tax, kind: component, head: income_tax} + - {variable: ni_employee, kind: measure_within, head: nics_individual, note: "Class 1 employee; national_insurance adds the classes directly."} + - {variable: ni_self_employed, kind: measure_within, head: nics_individual, note: "Classes 2 and 4; national_insurance adds the classes directly."} + - {variable: ni_class_1_employee, kind: component, head: nics_individual} + - {variable: ni_class_1_employee_primary, kind: component, head: nics_individual} + - {variable: ni_class_1_employee_additional, kind: component, head: nics_individual} + - {variable: ni_class_2, kind: component, head: nics_individual} + - {variable: ni_class_3, kind: component, head: nics_individual} + - {variable: ni_class_4, kind: component, head: nics_individual} + - {variable: ni_class_1_employer, kind: component, head: nics_employer} + - {variable: salary_sacrifice_pension_ni_employee, kind: measure_within, head: nics_individual, note: "The excess sacrifice is added back to employment_income, so Class 1 already charges it."} + - {variable: salary_sacrifice_pension_ni_employer, kind: measure_within, head: nics_employer, note: "As above, for employer Class 1."} + - {variable: tax, kind: composite, spans: [income_tax, nics_individual]} + - {variable: benunit_tax, kind: composite, spans: [income_tax, nics_individual]} + - {variable: total_national_insurance, kind: composite, spans: [nics_individual, nics_employer]} + - {variable: student_loan_repayment, kind: component, head: student_loan_repayments} + # Consumption + - {variable: baseline_vat, kind: baseline_measure, head: vat_change, note: "Subtracted inside vat_change."} + - {variable: beer_duty, kind: component, head: alcohol_duty} + - {variable: cider_duty, kind: component, head: alcohol_duty} + - {variable: sparkling_cider_duty, kind: component, head: alcohol_duty} + - {variable: wine_duty, kind: component, head: alcohol_duty} + - {variable: spirits_duty, kind: component, head: alcohol_duty} + - {variable: other_fermented_duty, kind: component, head: alcohol_duty} + - {variable: car_vehicle_excise_duty, kind: component, head: vehicle_excise_duty} + # Council tax net of a benefit + - {variable: council_tax_less_benefit, kind: composite, spans: [council_tax], note: "Council tax minus Council Tax Reduction, a benefit; not a receipt."} + +model_double_counts: + - variable: corporate_sdlt + contained_in: expected_sdlt + totals: [gov_tax, household_tax, pre_budget_change_household_tax] + issue: https://github.com/PolicyEngine/policyengine-uk/issues/2159 + +overlap_guards: + - id: sdlt_payer_split + kind: payer_split + heads: [sdlt_household] + property: >- + Until the #2159 fix, expected_sdlt adds corporate_sdlt, which the totals + also list (model_double_counts). The fix splits the head in two, neither + reading the other, both zeroed by gov.hmrc.stamp_duty.abolish. + - id: property_transaction_tax_geography + kind: geographic_exclusivity + heads: [sdlt_household, land_transaction_tax, land_and_buildings_transaction_tax] + property: >- + At most one is non-zero for any household (England and Northern Ireland, + Wales, Scotland). Until the #2159 fix this does not hold: expected_sdlt + carries a stray 0.045, and corporate SDLT for shareholders, in every + nation. + - id: local_tax_geography + kind: geographic_exclusivity + heads: [council_tax, domestic_rates] + property: >- + No household has both. Both are inputs, so this is a property of the + data, checked on the certified dataset. + - id: vat_level_and_change + kind: level_and_change + heads: [vat, vat_change] + property: Never in the same total. vat_change reads vat. + - id: employer_nics_level_and_incidence + kind: level_and_change + heads: [nics_employer, employer_nics_response_capital, employer_nics_response_consumer] + property: The level is only in gov_tax; the response incidence is only in household_tax. + - id: corporate_incidence_allocations + kind: shared_allocation_key + heads: [business_rates, sdlt_household, corporate_incident_tax_change, carbon_tax, land_value_tax] + property: >- + Each allocates a different receipt by shareholding; none reads another. + corporate_tax_incidence sums two of them and is excluded. + - id: private_school_vat_and_vat + kind: possible_base_overlap + heads: [vat, private_school_vat] + property: >- + vat applies to imputed consumption; if that consumption includes school + fees, turning on private_school_vat would tax them twice. The head is + zero at baseline, so baseline totals are unaffected; check the + consumption imputation before costing a reform that sets it. + - id: eligibility_reads + kind: read_not_inclusion + heads: [tv_licence, private_school_vat] + property: >- + These heads reach other heads of the same total only through + eligibility or imputation (see each head's reads). Raising the other + head by a small amount leaves them unchanged on the test population. + +spending_heads: + # Pensioners + - id: state_pension + label: State Pension + variable: state_pension + entity: person + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: pension_credit + label: Pension Credit + variable: pension_credit + entity: benunit + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: winter_fuel_payment + label: Winter Fuel Payment + variable: winter_fuel_allowance + entity: household + report_group: pensioner_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: pawhp + label: Pension Age Winter Heating Payment (Scotland) + variable: pawhp + entity: household + report_group: pensioner_benefits + totals: [gov_spending] + sign: nonnegative + model_note: >- + Counted in gov_spending but not in household_benefits or household net + income (PR 2047). + # Working-age means-tested benefits + - id: universal_credit + label: Universal Credit + variable: universal_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: housing_benefit + label: Housing Benefit + variable: housing_benefit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: council_tax_reduction + label: Council Tax Reduction + variable: council_tax_benefit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: esa_income + label: Employment and Support Allowance (income-related) + variable: esa_income + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: esa_contrib + label: Employment and Support Allowance (contribution-based) + variable: esa_contrib + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: jsa_income + label: Jobseeker's Allowance (income-based) + variable: jsa_income + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: jsa_contrib + label: Jobseeker's Allowance (contribution-based) + variable: jsa_contrib + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: income_support + label: Income Support + variable: income_support + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: working_tax_credit + label: Working Tax Credit + variable: working_tax_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: child_tax_credit + label: Child Tax Credit + variable: child_tax_credit + entity: benunit + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: incapacity_benefit + label: Incapacity Benefit + variable: incapacity_benefit + entity: person + report_group: working_age_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Disability and carers + - id: pip + label: Personal Independence Payment + variable: pip + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: dla + label: Disability Living Allowance + variable: dla + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: attendance_allowance + label: Attendance Allowance + variable: attendance_allowance + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: carers_allowance + label: Carer's Allowance + variable: carers_allowance + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: carer_support_payment + label: Carer Support Payment (Scotland) + variable: carer_support_payment + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: scottish_carer_supplement + label: Carer's Allowance Supplement (Scotland) + variable: scottish_carer_supplement + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: sda + label: Severe Disablement Allowance + variable: sda + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: iidb + label: Industrial Injuries Disablement Benefit + variable: iidb + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: afcs + label: Armed Forces Compensation Scheme + variable: afcs + entity: person + report_group: disability_and_carers_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Children and families + - id: child_benefit + label: Child Benefit + variable: child_benefit + entity: benunit + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: scottish_child_payment + label: Scottish Child Payment + variable: scottish_child_payment + entity: person + report_group: child_and_family_benefits + totals: [household_benefits] + sign: nonnegative + model_note: >- + Counted in household_benefits but missing from gov_spending, so + gov_balance leaves out its cost (about £0.5bn a year on uk-data 1.58.0). + - id: two_child_limit_payment + label: Two Child Limit Payment (Scotland, counterfactual) + variable: two_child_limit_payment + entity: benunit + report_group: child_and_family_benefits + totals: [household_benefits] + sign: nonnegative + zero_at_baseline: expected + model_note: Counted in household_benefits but not in gov_spending. + - id: healthy_start_vouchers + label: Healthy Start vouchers + variable: healthy_start_vouchers + entity: person + report_group: child_and_family_benefits + totals: [household_benefits] + sign: nonnegative + model_note: Counted in household_benefits but missing from gov_spending. + - id: maternity_allowance + label: Maternity Allowance + variable: maternity_allowance + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_maternity_pay + label: Statutory Maternity Pay + variable: statutory_maternity_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: statutory_sick_pay + label: Statutory Sick Pay + variable: statutory_sick_pay + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: ssmg + label: Sure Start Maternity Grant + variable: ssmg + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: bsp + label: Bereavement Support Payment + variable: bsp + entity: person + report_group: child_and_family_benefits + totals: [gov_spending, household_benefits] + sign: nonnegative + # Childcare and education support + - id: tax_free_childcare + label: Tax-Free Childcare + variable: tax_free_childcare + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: extended_childcare_entitlement + label: Extended childcare entitlement + variable: extended_childcare_entitlement + entity: benunit + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: universal_childcare_entitlement + label: Universal childcare entitlement + variable: universal_childcare_entitlement + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: targeted_childcare_entitlement + label: Targeted childcare entitlement + variable: targeted_childcare_entitlement + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: care_to_learn + label: Care to Learn + variable: care_to_learn + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: childcare_grant + label: Childcare Grant + variable: childcare_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: parents_learning_allowance + label: Parents' Learning Allowance + variable: parents_learning_allowance + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: adult_dependants_grant + label: Adult Dependants' Grant + variable: adult_dependants_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: travel_grant + label: Travel Grant + variable: travel_grant + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: disabled_students_allowance + label: Disabled Students' Allowance + variable: disabled_students_allowance + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + - id: bursary_fund_16_to_19 + label: 16 to 19 Bursary Fund + variable: bursary_fund_16_to_19 + entity: person + report_group: childcare_and_education + totals: [gov_spending, household_benefits] + sign: nonnegative + # Public services in kind + - id: nhs_spending + label: NHS spending + variable: nhs_spending + entity: person + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + - id: dfe_education_spending + label: State education spending + variable: dfe_education_spending + entity: household + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + - id: dft_subsidy_spending + label: Transport subsidy spending + variable: dft_subsidy_spending + entity: household + report_group: public_services_in_kind + totals: [gov_spending, household_benefits] + sign: nonnegative + caseload: false + # Cost-of-living and energy support (none in force at current law) + - id: cost_of_living_support_payment + label: Cost-of-living support payment + variable: cost_of_living_support_payment + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: energy_bills_rebate + label: Energy Bills Rebate + variable: energy_bills_rebate + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + - id: epg_subsidy + label: Energy Price Guarantee subsidy + variable: epg_subsidy + entity: household + report_group: cost_of_living_support + totals: [gov_spending, household_benefits] + sign: nonnegative + zero_at_baseline: expected + # Reform-only spending + - id: basic_income + label: Basic income + variable: basic_income + entity: person + report_group: hypothetical_spending + totals: [gov_spending, household_benefits] + sign: any + zero_at_baseline: expected + - id: disability_basic_income + label: Disability basic income + variable: disability_basic_income + entity: person + report_group: hypothetical_spending + totals: [gov_spending, household_benefits] + sign: any + zero_at_baseline: expected + - id: other_public_spending_budget_change + label: Other public spending budget change + variable: other_public_spending_budget_change + entity: household + report_group: hypothetical_spending + totals: [gov_spending] + sign: any + zero_at_baseline: expected + caseload: false diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py new file mode 100644 index 0000000000..82ad10326c --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_certified.py @@ -0,0 +1,198 @@ +"""The aggregate-impact measurement on the certified dataset. + +One year, the counted-once check on every tax component plus a few benefits. +Runs where microsimulation tests run (see conftest's data mode). +""" + +import dataclasses + +import numpy as np +import pytest + +from policyengine_uk.utils.aggregate_impact.measure import measure +from policyengine_uk.utils.aggregate_impact.partition import ( + default_partition_path, + load_partition, +) +from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parameter_node +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +pytestmark = pytest.mark.microsimulation + +YEAR = 2026 +# Universal Credit deducts income tax and National Insurance on Statutory Sick +# Pay (taxable_employment_income adds employment_benefits) but does not count +# the pay itself as earnings (uc_individual_earned_income_before_mif), so UC +# rises when SSP does. When that is fixed this test fails: remove the entry. +KNOWN_RISING_WITH_INCOME = {"statutory_sick_pay"} +SPENDING_SAMPLE = { + "state_pension", + "universal_credit", + "child_benefit", + "statutory_sick_pay", +} + + +@pytest.fixture(scope="module") +def measured(): + from policyengine_uk import Microsimulation + + sim = Microsimulation() + dataset = sim.dataset + partition = load_partition(default_partition_path()) + spec = dataclasses.replace( + load_spec(), first_year=YEAR, last_year=YEAR, guard_year=YEAR + ) + tax_components = {h.variable for h in partition.tax_heads} | { + d.variable for d in partition.double_counts + } + result, households = measure( + sim, + lambda reform=None: Microsimulation(dataset=dataset, reform=reform), + partition, + spec, + label="test", + reforms=[r for r in load_reforms() if r.scale], + guard_variables=tax_components | SPENDING_SAMPLE, + ) + return result, households, partition, Microsimulation(dataset=dataset) + + +def checks(result, prefix): + return [c for c in result["checks"] if c["id"].startswith(prefix)] + + +def test_accounting_identities_and_closure_hold(measured): + result, _, _, _ = measured + assert result["status"] == "ok" + for prefix in ( + "closure", + "identity:", + "finite:", + "weights:", + "bounds:", + "determinism:", + ): + failing = [c for c in checks(result, prefix) if c["status"] != "pass"] + assert failing == [], failing + + +def test_every_tax_component_is_counted_exactly_once(measured): + result, _, partition, _ = measured + guard = result["guard"] + assert guard["status"] == "ran" and guard["reset_reproduces_baseline"] + rows = {r["variable"]: r for r in guard["components"]} + for head in partition.tax_heads: + row = rows[head.variable] + assert row["status"] == "pass", row + for total, k in row["coefficients"].items(): + assert k == pytest.approx(1, abs=1e-3), (head.id, total) + + +def test_declared_double_counts_are_still_double_counted_by_the_model(measured): + result, _, partition, _ = measured + rows = {r["variable"]: r for r in result["guard"]["components"]} + for declared in partition.double_counts: + present = next( + d + for d in result["guard"]["double_counts"] + if d["variable"] == declared.variable + ) + assert present["status"] == "pass", present + for total in declared.totals: + assert rows[declared.variable]["model_coefficients"][ + total + ] == pytest.approx(2, abs=1e-3) + + +def test_spending_never_rises_by_more_than_is_added_except_known_defects(measured): + result, _, _, _ = measured + rows = {r["variable"]: r for r in result["guard"]["components"]} + for variable in SPENDING_SAMPLE: + row = rows[variable] + if variable in KNOWN_RISING_WITH_INCOME: + assert ( + row["status"] == "fail" + and 1 < row["coefficients"]["gov_spending"] < 1.05 + ) + else: + assert row["status"] == "pass", row + + +def test_head_totals_agree_with_the_model_weighted_sums(measured): + result, _, partition, sim = measured + metrics = {m["id"]: m for m in result["metrics"]} + for head in partition.heads: + section = "receipts" if head.kind == "tax" else "spending" + ours = metrics[f"{section}.{head.id}"]["values"][str(YEAR)] + theirs = float(sim.calculate(head.variable, YEAR).sum()) + assert ours == pytest.approx(theirs, rel=1e-6, abs=1e3), head.id + receipts = metrics["balance.receipts"]["values"][str(YEAR)] + model = metrics["model.gov_tax"]["values"][str(YEAR)] + double_counted = sum( + metrics[f"double_counts.{d.variable}"]["values"][str(YEAR)] + for d in partition.double_counts + if "gov_tax" in d.totals + ) + assert receipts + double_counted == pytest.approx(model, rel=1e-9) + + +def test_household_detail_is_complete(measured): + _, households, _, _ = measured + columns = households[str(YEAR)] + n = len(columns["household_id"]) + assert all(len(v) == n for v in columns.values()) + assert np.all(np.isfinite(columns["income"])) + assert set(np.unique(columns["decile"])) <= set(range(-1, 11)) + + +def test_relative_reforms_scale_their_parameters_and_move_their_targets(measured): + result, _, _, sim = measured + passing = {c["id"] for c in result["checks"] if c["status"] == "pass"} + relative = [r for r in load_reforms() if r.scale] + assert {r.id for r in relative} == { + "uc_standard_allowance_plus_10pc", + "state_pension_plus_10pc", + } + outcomes = {o["id"]: o for o in result["reforms"]} + for reform in relative: + assert f"reform_runs:{reform.id}" in passing + assert f"responsive:{reform.id}" in passing + for path, factor in reform.scale.items(): + baseline = parameter_node(sim.tax_benefit_system.parameters, path)( + f"{YEAR}-01-01" + ) + assert outcomes[reform.id]["parameters"][path][str(YEAR)] == pytest.approx( + baseline * factor + ) + + +def test_the_published_result_holds_aggregates_only(measured): + """result.json may be public: no household-length arrays, no single values.""" + result, households, _, _ = measured + n_households = len(households[str(YEAR)]["household_id"]) + + def numeric_lists(node): + if isinstance(node, dict): + for value in node.values(): + yield from numeric_lists(value) + elif isinstance(node, list): + if node and all(isinstance(v, (int, float)) for v in node): + yield node + for value in node: + yield from numeric_lists(value) + + longest = max((len(values) for values in numeric_lists(result)), default=0) + assert longest <= 50 < n_households + details = " ".join(d for c in result["checks"] for d in c["details"]) + assert "minimum £" not in details and "largest gap" not in details + + +def test_geographic_guards_hold_except_where_a_double_count_is_declared(measured): + """Council tax and domestic rates never meet. SDLT, LTT and LBTT overlap + only while expected_sdlt carries corporate SDLT (#2159).""" + result, _, partition, _ = measured + status = {c["id"]: c["status"] for c in result["checks"]} + assert status["overlap_guard:local_tax_geography"] == "pass" + overlapping = status["overlap_guard:property_transaction_tax_geography"] == "fail" + assert overlapping == bool(partition.double_counts) diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py new file mode 100644 index 0000000000..b408f40cd6 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_compare.py @@ -0,0 +1,418 @@ +"""Comparison and rendering of two measurements, on synthetic results.""" + +import copy +import math + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact import COMMENT_MARKER, TOOL_SCHEMA +from policyengine_uk.utils.aggregate_impact.compare import ( + REQUIRED_CHECKS, + _match, + classify_check, + compare, + decile_impacts, +) +from policyengine_uk.utils.aggregate_impact.measure import weighted_gini +from policyengine_uk.utils.aggregate_impact.render import fmt_change, fmt_level, render +from policyengine_uk.utils.aggregate_impact.spec import Threshold, load_spec + +SPEC = load_spec() +YEARS = SPEC.years +SHA = "5" * 64 + +KINDS = { + "receipts": "gbp", + "caseloads": "count", + "poverty": "rate", + "inequality": "index", +} +finite = st.floats(-1e12, 1e12, allow_nan=False, allow_infinity=False) + + +def check(check_id, status="pass", failing=()): + return { + "id": check_id, + "label": check_id, + "section": "s", + "severity": "fail", + "status": status, + "details": ["d"] if status != "pass" else [], + "failing": { + scope: "d" for scope in (failing or ([""] if status == "fail" else [])) + }, + } + + +def result(values, checks=(), sha=SHA, status="ok", partition_sha="p", required=True): + """A measurement with one metric per (section, name) and the given checks.""" + all_checks = [check(c) for c in REQUIRED_CHECKS] if required else [] + for entry in checks: + all_checks.append(check(*entry) if isinstance(entry, tuple) else entry) + return { + "schema": TOOL_SCHEMA, + "status": status, + "label": "x", + "years": YEARS, + "provenance": { + "dataset": {"verified_sha256": sha, "path": "d.h5"}, + "git_sha": "abc", + }, + "runtime": { + "distributions": ["numpy==2.0"], + "python": "3.13", + "machine": "arm64", + }, + "partition": {"sha256": partition_sha, "source": "s"}, + "metrics": [ + { + "id": f"{section}.{name}", + "label": name, + "section": section, + "group": None, + "kind": KINDS[section], + "unit": None, + "values": {str(y): v for y, v in zip(YEARS, series)}, + } + for (section, name), series in values.items() + ], + "checks": all_checks, + "guard": {"status": "skipped", "reason": "test"}, + "reforms": [], + "timings": {"total": 1}, + } + + +metric_values = st.dictionaries( + st.tuples(st.sampled_from(sorted(KINDS)), st.sampled_from(["a", "b", "c"])), + st.lists(finite, min_size=len(YEARS), max_size=len(YEARS)), + min_size=1, + max_size=6, +) + + +@pytest.mark.parametrize( + "base, head, blocking, expected", + [ + ("pass", "pass", False, "pass"), + ("fail", "pass", False, "fixed"), + ("warn", "pass", False, "fixed"), + (None, "pass", False, "pass"), + ("pass", "fail", False, "regression"), + (None, "fail", False, "regression"), + ("skipped", "fail", False, "regression"), + ("fail", "fail", False, "pre_existing"), + ("fail", "fail", True, "regression"), + ("warn", "fail", False, "regression"), + ("pass", "warn", False, "new_warning"), + ("warn", "warn", False, "warning"), + ("pass", "skipped", False, "skipped"), + ("pass", None, False, "removed"), + ], +) +def test_check_classification(base, head, blocking, expected): + assert classify_check(base, head, blocking) == expected + + +@settings(max_examples=200, deadline=None) +@given( + base=st.sets(st.sampled_from(["2025", "2026", "2027", ""])), + head=st.sets(st.sampled_from(["2025", "2026", "2027", ""]), min_size=1), +) +def test_a_failure_is_pre_existing_only_if_the_base_fails_everywhere_the_head_does( + base, head +): + verdict = classify_check("fail" if base else "pass", "fail", False, base, head) + assert verdict == ("pre_existing" if base and head <= base else "regression") + + +@settings(max_examples=60, deadline=None) +@given(values=metric_values) +def test_identical_sides_change_nothing(values): + side = result(values, checks=[("identity:x", "pass"), ("sign:y", "fail")]) + summary = compare(side, copy.deepcopy(side), SPEC) + assert summary["status"] == "pass", summary["failures"] + assert summary["flags"] == [] + assert all(not row["changed"] for row in summary["rows"]) + assert {c["classification"] for c in summary["checks"]} == {"pass", "pre_existing"} + + +@settings(max_examples=60, deadline=None) +@given(base=metric_values, data=st.data()) +def test_changes_are_antisymmetric(base, data): + head = {k: [v + data.draw(finite) for v in series] for k, series in base.items()} + forward = {r["id"]: r for r in compare(result(base), result(head), SPEC)["rows"]} + backward = {r["id"]: r for r in compare(result(head), result(base), SPEC)["rows"]} + for metric_id, row in forward.items(): + for year, cell in row["values"].items(): + assert cell["delta"] == pytest.approx( + -backward[metric_id]["values"][year]["delta"], abs=1e-3 + ) + + +@settings(max_examples=200, deadline=None) +@given(base=finite, head=finite, absolute=st.floats(0, 1e9), relative=st.floats(0, 0.5)) +def test_a_move_is_flagged_exactly_when_it_exceeds_the_larger_bound( + base, head, absolute, relative +): + threshold = Threshold(absolute, relative) + assert threshold.exceeded(base, head) == ( + abs(head - base) > max(absolute, relative * abs(base)) + ) + + +def test_flags_follow_the_spec_thresholds(): + base = {("receipts", "a"): [100e9] * len(YEARS)} + small = {("receipts", "a"): [100e9 + 50e6] * len(YEARS)} + large = {("receipts", "a"): [100e9 + 200e6] * len(YEARS)} + assert compare(result(base), result(small), SPEC)["flags"] == [] + assert len(compare(result(base), result(large), SPEC)["flags"]) == len(YEARS) + + +CRASHED = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "error": "Traceback\nValueError: boom", +} +ONE = {("receipts", "a"): [1.0] * len(YEARS)} + + +def test_a_crashed_or_missing_head_fails(): + summary = compare(result(ONE), CRASHED, SPEC) + assert summary["status"] == "fail" and "boom" in summary["failures"][0] + assert compare(result(ONE), None, SPEC)["status"] == "fail" + + +def test_a_crashed_or_missing_base_fails_unless_it_is_not_required(): + for base in (CRASHED, None): + assert compare(base, result(ONE), SPEC)["status"] == "fail" + relaxed = compare(base, result(ONE), SPEC, require_base=False) + assert relaxed["status"] == "pass" and relaxed["warnings"] + + +def test_without_a_base_the_heads_own_failing_checks_block(): + head = result(ONE, checks=[("identity:x", "fail")]) + summary = compare(None, head, SPEC, require_base=False) + assert summary["status"] == "fail" + assert any("identity:x" in f for f in summary["failures"]) + + +def test_different_datasets_fail(): + summary = compare(result(ONE), result(ONE, sha="6" * 64), SPEC) + assert summary["status"] == "fail" + + +def test_non_finite_head_figures_block_even_when_the_base_shares_them(): + checks = [check("finite:x", "fail")] + summary = compare(result(ONE, checks), result(ONE, checks), SPEC) + assert summary["status"] == "fail" + + +def test_a_non_finite_head_figure_fails_even_if_no_check_caught_it(): + """Reform costings are measured after most checks; compare looks itself.""" + bad = {("receipts", "a"): [1.0] * (len(YEARS) - 1) + [float("nan")]} + summary = compare(result(ONE), result(bad), SPEC) + assert summary["status"] == "fail" + assert any("not finite" in f for f in summary["failures"]) + + +def test_a_newly_failing_year_is_a_regression_not_pre_existing(): + base = result(ONE, checks=[check("identity:x", "fail", ["2025"])]) + same = result(ONE, checks=[check("identity:x", "fail", ["2025"])]) + worse = result(ONE, checks=[check("identity:x", "fail", ["2025", "2026"])]) + assert compare(base, same, SPEC)["status"] == "pass" + summary = compare(base, worse, SPEC) + assert summary["status"] == "fail" + assert any("identity:x" in f for f in summary["failures"]) + + +def test_a_check_that_vanishes_fails_unless_the_partition_changed(): + base = result(ONE, checks=[("sign:vat", "pass")]) + summary = compare(base, result(ONE), SPEC) + assert summary["status"] == "fail" + assert any("sign:vat" in f for f in summary["failures"]) + edited = compare(base, result(ONE, partition_sha="q"), SPEC) + assert edited["status"] == "pass" + assert any("sign:vat" in w for w in edited["warnings"]) + + +def test_a_head_without_the_required_checks_fails(): + summary = compare(result(ONE), result(ONE, required=False), SPEC) + assert summary["status"] == "fail" + assert any("finite:metrics" in f for f in summary["failures"]) + + +def test_a_check_that_no_longer_applies_is_noted_not_failed(): + base = result(ONE, checks=[("counted_once:x", "pass")]) + head = result(ONE, checks=[check("counted_once:x", "skipped")]) + summary = compare(base, head, SPEC) + assert summary["status"] == "pass" + assert any("counted_once:x" in w for w in summary["warnings"]) + + +def test_a_figure_on_one_side_only_is_a_change(): + two = {**ONE, ("receipts", "b"): [2.0] * len(YEARS)} + removed = compare(result(two), result(ONE), SPEC) + row = next(r for r in removed["rows"] if r["id"] == "receipts.b") + assert row["changed"] and row["only_in"] == "base" + text = render(removed, SPEC) + assert "removed: base levels" in text + assert "No change in any year:** receipts" not in text + added = compare(result(ONE), result(two), SPEC) + assert "new: head levels" in render(added, SPEC) + + +def test_strict_mode_fails_pre_existing_failures(): + side = result(ONE, checks=[("sign:vat", "fail")]) + assert compare(side, copy.deepcopy(side), SPEC)["status"] == "pass" + assert compare(side, copy.deepcopy(side), SPEC, strict=True)["status"] == "fail" + + +def test_a_partition_edit_is_noted(): + summary = compare(result(ONE), result(ONE, partition_sha="q"), SPEC) + assert any("partition" in w for w in summary["warnings"]) + + +def test_households_match_only_one_to_one(): + assert _match(np.array([1, 2]), np.array([1, 2, 3])) is None + assert _match(np.array([1, 2]), np.array([1, 1])) is None + assert _match(np.array([1, 1]), np.array([1, 1])) is None + assert np.array_equal(_match(np.array([1, 2, 3]), np.array([3, 1, 2])), [1, 2, 0]) + + +@settings(max_examples=100, deadline=None) +@given( + ids=st.lists(st.integers(0, 10_000), min_size=1, max_size=60, unique=True), + data=st.data(), +) +def test_matching_recovers_any_permutation(ids, data): + base = np.array(ids) + head = np.array(data.draw(st.permutations(ids))) + matched = _match(base, head) + assert matched is not None and np.array_equal(head[matched], base) + + +def households(n, rng, decile=None, income=None, weight=None): + return { + str(y): { + "household_id": np.arange(n), + "household_weight": weight + if weight is not None + else rng.uniform(1, 1000, n), + "household_count_people": rng.integers(1, 6, n).astype(float), + "decile": decile if decile is not None else rng.integers(1, 11, n), + "income": income if income is not None else rng.uniform(1e3, 1e5, n), + } + for y in YEARS + } + + +@settings(max_examples=30, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), n=st.integers(20, 400)) +def test_decile_impacts_match_a_direct_grouped_calculation(seed, n): + rng = np.random.default_rng(seed) + base = households(n, rng) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + rng.normal(0, 500, n) + impacts, _ = decile_impacts(base, head, YEARS) + year = str(YEARS[0]) + b, h = base[year], head[year] + frame = pd.DataFrame( + { + "d": b["decile"], + "w": b["household_weight"], + "base": b["income"], + "change": h["income"] - b["income"], + } + ) + for row in impacts[year][:-1]: + group = frame[frame.d == row["decile"]] + if group.empty: + continue + mean_change = (group.w * group.change).sum() / group.w.sum() + relative = (group.w * group.change).sum() / (group.w * group.base).sum() + assert row["mean_change"] == pytest.approx(mean_change) + assert row["relative_change"] == pytest.approx(relative) + + +@settings(max_examples=30, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), shift=st.floats(-1e4, 1e4)) +def test_a_uniform_change_shows_in_every_decile(seed, shift): + rng = np.random.default_rng(seed) + base = households(200, rng, decile=np.repeat(np.arange(1, 11), 20)) + head = copy.deepcopy(base) + for columns in head.values(): + columns["income"] = columns["income"] + shift + columns["household_id"] = columns["household_id"][::-1].copy() + for key in ("household_weight", "household_count_people", "decile", "income"): + columns[key] = columns[key][::-1].copy() + impacts, notes = decile_impacts(base, head, YEARS) + assert not any("differ" in n for n in notes) + for row in impacts[str(YEARS[0])]: + assert row["mean_change"] == pytest.approx(shift, abs=1e-6) + assert row["share_gaining"] == (1.0 if shift > 1 else 0.0) + assert row["share_losing"] == (1.0 if shift < -1 else 0.0) + + +@settings(max_examples=50, deadline=None) +@given(seed=st.integers(0, 2**32 - 1), n=st.integers(2, 300)) +def test_gini_matches_microdf(seed, n): + from microdf import MicroSeries + + rng = np.random.default_rng(seed) + income = rng.lognormal(10, 1, n) + weights = rng.uniform(0.5, 50, n) + assert weighted_gini(income, weights) == pytest.approx( + float(MicroSeries(income, weights=weights).gini()), abs=1e-6 + ) + + +def test_gini_bounds(): + assert weighted_gini(np.ones(10), np.ones(10)) == pytest.approx(0) + assert weighted_gini(np.r_[np.zeros(99), 1.0], np.ones(100)) == pytest.approx(0.99) + + +@pytest.mark.parametrize( + "value, kind, text", + [ + (1.234e9, "gbp", "+£1.23bn"), + (-5.5e6, "gbp", "-£5.5m"), + (0, "gbp", "0"), + (12_300, "count", "+12.3k"), + (0.0012, "rate", "+0.12pp"), + (None, "gbp", "n/a"), + ], +) +def test_change_formatting(value, kind, text): + assert fmt_change(value, kind) == text + + +def test_level_formatting(): + assert fmt_level(320.5e9, "gbp") == "£320.50bn" + assert fmt_level(0.2335, "rate") == "23.4%" + assert fmt_level(float("nan"), "gbp") == "n/a" + + +@settings(max_examples=20, deadline=None) +@given(values=metric_values, limit=st.integers(2000, 20000)) +def test_the_comment_carries_the_marker_and_fits_its_limit(values, limit): + head = {k: [v + 1e9 for v in series] for k, series in values.items()} + summary = compare(result(values), result(head), SPEC) + spec = SPEC.__class__(**{**SPEC.__dict__, "max_comment_characters": limit}) + text = render(summary, spec) + assert text.startswith(COMMENT_MARKER) + assert len(text) <= limit + 1 + assert render(summary, spec) == text + + +def test_a_skipped_head_passes_with_a_note(): + skipped = {"schema": TOOL_SCHEMA, "status": "skipped", "reason": "no data access"} + summary = compare(None, skipped, SPEC) + assert summary["status"] == "skipped" and summary["failures"] == [] + text = render(summary, SPEC) + assert text.startswith(COMMENT_MARKER) and "no data access" in text diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py new file mode 100644 index 0000000000..5268196c19 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_data_mode.py @@ -0,0 +1,90 @@ +"""Mandatory-data mode end to end, in a subprocess with this repo's hooks. + +A scratch test directory imports the real conftest's hooks, so the cases +exercise the hook ordering pytest actually applies. +""" + +import os +import subprocess +import sys +import textwrap + +import pytest + +CONFTEST = "from policyengine_uk.tests.conftest import * # noqa: F401,F403\n" +CASES = textwrap.dedent( + """ + import pytest + + def test_needs_no_data(): + assert True + + @pytest.mark.microsimulation + def test_needs_data(): + assert True + + @pytest.mark.microsimulation + @pytest.mark.skip(reason="a skip marker must not hide missing data") + def test_needs_data_but_is_marked_skip(): + assert True + """ +) + + +def run(tmp_path, **env_changes): + (tmp_path / "conftest.py").write_text(CONFTEST) + (tmp_path / "test_cases.py").write_text(CASES) + env = { + k: v + for k, v in os.environ.items() + if k + not in ( + "HUGGING_FACE_TOKEN", + "POLICYENGINE_UK_DEFAULT_DATASET", + "POLICYENGINE_UK_REQUIRE_TEST_DATA", + "PYTEST_ADDOPTS", + ) + } + env.update(env_changes) + return subprocess.run( + [ + sys.executable, + "-m", + "pytest", + str(tmp_path), + "-q", + "-p", + "no:cacheprovider", + "-p", + "no:xdist", + "--rootdir", + str(tmp_path), + "-W", + "ignore::pytest.PytestUnknownMarkWarning", + ], + env=env, + capture_output=True, + text=True, + timeout=600, + ) + + +def test_without_data_the_data_tests_skip_and_the_rest_run(tmp_path): + done = run(tmp_path) + assert done.returncode == 0, done.stdout + done.stderr + assert "1 passed, 2 skipped" in done.stdout + + +def test_in_mandatory_mode_every_data_test_fails_even_one_marked_skip(tmp_path): + done = run(tmp_path, POLICYENGINE_UK_REQUIRE_TEST_DATA="1") + assert done.returncode != 0 + assert "1 passed, 2 errors" in done.stdout, done.stdout + assert "skipped" not in done.stdout.splitlines()[-1] + assert "Mandatory-data mode" in done.stdout + + +@pytest.mark.parametrize("value", ["maybe", "2"]) +def test_an_unrecognised_mode_value_fails_closed(tmp_path, value): + done = run(tmp_path, POLICYENGINE_UK_REQUIRE_TEST_DATA=value) + assert done.returncode != 0 + assert "2 errors" in done.stdout, done.stdout diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py new file mode 100644 index 0000000000..397fcfecc2 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_dataset.py @@ -0,0 +1,191 @@ +"""The certified dataset pin, the fixtures that record it, and the data mode.""" + +import hashlib +import os +from pathlib import Path + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.tests.conftest import ( + CERTIFIED_DATASET_PIN, + DATASET_ENV_VAR, + DEFAULT_TEST_DATASET_URL, + REQUIRE_DATA_ENV_VAR, + TOKEN_ENV_VAR, + plan_test_data, +) +from policyengine_uk.utils.aggregate_impact.dataset import ( + DatasetMismatchError, + DatasetPinError, + load_pin, + sha256_file, + verify_file, +) + +TESTS = Path(__file__).resolve().parents[1] +PIN = load_pin(CERTIFIED_DATASET_PIN) + + +def test_pin_is_immutable_and_complete(): + assert PIN.problems() == [] + assert len(PIN.revision) == 40 and len(PIN.sha256) == 64 + assert PIN.uri == f"hf://{PIN.repo_id}/{PIN.path}@{PIN.revision}" + assert DEFAULT_TEST_DATASET_URL == PIN.uri + + +def test_reform_fixtures_record_the_pinned_dataset(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f) + recorded = fixtures["dataset"] + assert recorded["revision"] == PIN.revision + assert recorded["sha256"] == PIN.sha256 + assert recorded["path"] == PIN.path + assert str(recorded["data_version"]) == PIN.data_version + + +def test_every_reform_fixture_keeps_its_receipts(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f) + for reform in fixtures["reforms"]: + receipts = reform["receipts"] + assert len(receipts) >= 2, reform["name"] + latest = receipts[-1] + assert PIN.data_version in latest["data"], reform["name"] + assert round(latest["impact"], 1) == reform["expected_impact"], reform["name"] + + +def _pin_file(tmp_path, **overrides): + raw = { + "repo_id": PIN.repo_id, + "path": PIN.path, + "revision": PIN.revision, + "sha256": PIN.sha256, + **overrides, + } + path = tmp_path / "pin.yaml" + path.write_text(yaml.safe_dump(raw)) + return path + + +@pytest.mark.parametrize( + "overrides", + [ + {"revision": "1.58.0"}, + {"revision": "main"}, + {"sha256": "5BA399EA"}, + {"repo_id": "no-slash"}, + {"path": "/absolute.h5"}, + {"unexpected": "key"}, + ], +) +def test_mutable_or_malformed_pins_are_rejected(tmp_path, overrides): + with pytest.raises(DatasetPinError): + load_pin(_pin_file(tmp_path, **overrides)) + + +@settings(max_examples=50, deadline=None) +@given(revision=st.text(max_size=45)) +def test_only_a_full_commit_hash_is_accepted_as_a_revision(revision): + from dataclasses import replace + + pin = replace(PIN, revision=revision) + is_commit = len(revision) == 40 and all(c in "0123456789abcdef" for c in revision) + assert (not any("revision" in p for p in pin.problems())) == is_commit + + +@settings(max_examples=30, deadline=None) +@given(content=st.binary(max_size=4096), tamper=st.integers(0, 4095)) +def test_verification_accepts_the_pinned_bytes_and_rejects_any_other( + tmp_path_factory, content, tamper +): + path = tmp_path_factory.mktemp("data") / "file.h5" + path.write_bytes(content) + digest = hashlib.sha256(content).hexdigest() + assert sha256_file(path) == digest + assert verify_file(path, digest) == digest + if content: + changed = bytearray(content) + changed[tamper % len(content)] ^= 0xFF + path.write_bytes(bytes(changed)) + with pytest.raises(DatasetMismatchError): + verify_file(path, digest) + + +ENV_VALUES = st.one_of( + st.none(), st.sampled_from(["", "0", "1", "true", "FALSE", "yes", "maybe"]) +) +DATASETS = st.one_of( + st.none(), st.just(""), st.just(PIN.uri), st.just("hf://other/repo/file.h5@abc") +) +TOKENS = st.one_of(st.none(), st.just(""), st.just("hf_token")) + + +def _env(flag, dataset, token, extra): + env = dict(extra) + for key, value in ( + (REQUIRE_DATA_ENV_VAR, flag), + (DATASET_ENV_VAR, dataset), + (TOKEN_ENV_VAR, token), + ): + if value is not None: + env[key] = value + return env + + +@settings(max_examples=300, deadline=None) +@given( + flag=ENV_VALUES, + dataset=DATASETS, + token=TOKENS, + extra=st.dictionaries(st.sampled_from(["HOME", "CI", "PATH"]), st.text(max_size=5)), +) +def test_data_plan_invariants(flag, dataset, token, extra): + plan = plan_test_data(_env(flag, dataset, token, extra), PIN.uri) + assert plan.action in ("run", "skip", "fail") + # Mandatory mode never skips, and runs only on the pinned dataset. + if plan.mandatory: + assert plan.action != "skip" + if plan.action == "run": + assert plan.pinned and plan.dataset == PIN.uri + # Running always names a dataset; an explicit one always wins. + if plan.action == "run": + assert plan.dataset + if dataset: + assert plan.dataset == dataset + # Nothing configured and not mandatory: the old behaviour, a skip. + if ( + not dataset + and not token + and str(flag or "").strip().lower() in ("", "0", "false", "no") + ): + assert plan.action == "skip" + # An unrecognised mode value fails closed. + if flag is not None and flag.strip().lower() not in ( + "", + "0", + "1", + "true", + "false", + "yes", + "no", + "on", + "off", + ): + assert plan.action == "fail" + # Unrelated variables never change the plan. + assert plan == plan_test_data(_env(flag, dataset, token, {}), PIN.uri) + + +def test_data_plan_examples(): + assert plan_test_data({TOKEN_ENV_VAR: "t"}, PIN.uri).action == "run" + assert plan_test_data({}, PIN.uri).action == "skip" + assert plan_test_data({REQUIRE_DATA_ENV_VAR: "1"}, PIN.uri).action == "fail" + assert plan_test_data( + {REQUIRE_DATA_ENV_VAR: "1", TOKEN_ENV_VAR: "t"}, PIN.uri + ).pinned + other = {REQUIRE_DATA_ENV_VAR: "1", DATASET_ENV_VAR: "local.h5"} + assert plan_test_data(other, PIN.uri).action == "fail" + assert plan_test_data({DATASET_ENV_VAR: "local.h5"}, PIN.uri).action == "run" diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py new file mode 100644 index 0000000000..a4cd4cb2b4 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_guard.py @@ -0,0 +1,290 @@ +"""The counted-once check, on a small fake simulation whose formulas are known. + +The fake implements only the simulation API the check uses, so each case +controls exactly which component contains or reads which. +""" + +from types import SimpleNamespace + +import numpy as np +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact.guard import counted_once, snapshot_inputs +from policyengine_uk.utils.aggregate_impact.partition import parse_partition + +YEAR = 2026 + + +class FakeHolder: + def __init__(self, sim, variable): + self.sim, self.variable = sim, variable + + def get_known_periods(self): + return [p for (v, p) in self.sim.inputs if v == self.variable] + + def get_array(self, period): + return self.sim.inputs[(self.variable, str(period))] + + +class FakeSim: + """Household-level variables only; formulas read each other by name.""" + + def __init__(self, formulas, inputs, weights): + self.formulas = formulas + self.inputs = { + (k, str(YEAR)): np.asarray(v, dtype=float) for k, v in inputs.items() + } + self.inputs[("household_weight", str(YEAR))] = np.asarray(weights, dtype=float) + self.cache = {} + names = set(formulas) | set(inputs) | {"household_weight"} + self.tax_benefit_system = SimpleNamespace( + variables={ + name: SimpleNamespace( + entity=SimpleNamespace(key="household"), + definition_period="year", + value_type=float, + ) + for name in names + } + ) + + def calculate(self, variable, period, unweighted=True, map_to=None): + key = (variable, str(period)) + if key in self.inputs: + return self.inputs[key] + if key not in self.cache: + self.cache[key] = np.asarray( + self.formulas[variable](lambda name: self.calculate(name, period)), + dtype=float, + ) + return self.cache[key] + + def map_result(self, values, source, target): + assert source == target == "household" + return values + + def set_input(self, variable, period, values): + self.inputs[(variable, str(period))] = np.asarray(values, dtype=float) + + def delete_arrays(self, variable, period=None): + for store in (self.inputs, self.cache): + for key in [k for k in store if k[0] == variable]: + del store[key] + + def get_holder(self, variable): + return FakeHolder(self, variable) + + def _invalidate_all_caches(self): + self.cache = {} + + +def partition(tax_heads, spending_heads=(), double_counts=(), reads=None): + reads = reads or {} + + def heads(spec, total): + return [ + { + "id": v, + "label": v, + "variable": v, + "totals": [total], + "entity": "household", + "reads": [{"head": r} for r in reads.get(v, [])], + } + for v in spec + ] + + return parse_partition( + { + "schema_version": 1, + "totals": {"gov_tax": {}, "gov_spending": {}}, + "tax_heads": heads(tax_heads, "gov_tax"), + "spending_heads": heads(spending_heads, "gov_spending"), + "model_double_counts": [ + {"variable": v, "contained_in": c, "totals": ["gov_tax"]} + for v, c in double_counts + ], + } + ) + + +def model(contains=True, interaction=-0.5): + """tax_a may contain tax_b; ben_d reads ben_c with the given slope.""" + return { + "tax_b": lambda f: 5.0 + 0 * f("x"), + "tax_a": lambda f: 0.1 * f("x") + (f("tax_b") if contains else 0), + "gov_tax": lambda f: f("tax_a") + f("tax_b"), + "ben_c": lambda f: 100 + 0 * f("x"), + "ben_d": lambda f: np.maximum(0, 200 + interaction * f("ben_c")), + "gov_spending": lambda f: f("ben_c") + f("ben_d"), + } + + +def run(formulas, part, x=(1000, 2000, 3000), weights=(1.5, 2.0, 0.5)): + sim = FakeSim(formulas, {"x": x}, weights) + snapshot = snapshot_inputs(sim, part.component_variables()) + result = counted_once(sim, part, YEAR, 1e-6, snapshot) + return sim, result + + +def by_variable(result): + return {r["variable"]: r for r in result["components"]} + + +def test_declared_double_count_passes_and_the_model_total_counts_it_twice(): + _, result = run( + model(contains=True), + partition(["tax_a"], ["ben_c", "ben_d"], [("tax_b", "tax_a")]), + ) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "pass" + assert rows["tax_b"]["coefficients"]["gov_tax"] == pytest.approx(1) + assert rows["tax_b"]["model_coefficients"]["gov_tax"] == pytest.approx(2) + assert result["double_counts"][0]["status"] == "pass" + assert result["reset_reproduces_baseline"] + + +def test_stale_double_count_declaration_fails(): + _, result = run( + model(contains=False), partition(["tax_a"], [], [("tax_b", "tax_a")]) + ) + assert result["double_counts"][0]["status"] == "fail" + assert result["double_counts"][0]["coefficient"] == pytest.approx(0) + + +def test_undeclared_containment_fails_because_another_head_moves(): + _, result = run(model(contains=True), partition(["tax_a", "tax_b"])) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "fail" + assert "tax_a moves by +1.0000" in rows["tax_b"]["detail"] + assert rows["tax_b"]["responses"]["gov_tax"]["tax_a"] == pytest.approx(1) + assert rows["tax_a"]["status"] == "pass" + + +def test_means_tested_offset_passes_but_a_benefit_rising_with_income_fails(): + _, offset = run(model(interaction=-0.5), partition(["tax_a"], ["ben_c", "ben_d"])) + row = by_variable(offset)["ben_c"] + assert row["coefficients"]["gov_spending"] == pytest.approx(0.5) + assert row["status"] == "pass" + _, rising = run(model(interaction=0.2), partition(["tax_a"], ["ben_c", "ben_d"])) + row = by_variable(rising)["ben_c"] + assert row["status"] == "fail" and "ben_d rises by 0.2000" in row["detail"] + + +def test_a_total_that_adds_a_zero_component_twice_fails(): + """Baseline identities hold while tax_b is zero; the response gives it away.""" + formulas = model(contains=False) + formulas["tax_b"] = lambda f: 0 * f("x") + formulas["gov_tax"] = lambda f: f("tax_a") + 2 * f("tax_b") + _, result = run(formulas, partition(["tax_a", "tax_b"])) + row = by_variable(result)["tax_b"] + assert row["coefficients"]["gov_tax"] == pytest.approx(1) + assert row["model_coefficients"]["gov_tax"] == pytest.approx(2) + assert ( + row["status"] == "fail" + and "the model's gov_tax moves by 2.0000" in row["detail"] + ) + + +def test_a_means_test_fall_cannot_hide_a_benefit_that_contains_another(): + """ben_e contains ben_c while ben_d falls pound for pound: the net is 1.""" + formulas = model(interaction=-1.0) + formulas["ben_e"] = lambda f: 50 + f("ben_c") + formulas["gov_spending"] = lambda f: f("ben_c") + f("ben_d") + f("ben_e") + _, result = run(formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_e"])) + row = by_variable(result)["ben_c"] + assert row["coefficients"]["gov_spending"] == pytest.approx(1) + assert row["status"] == "fail" and "ben_e rises by 1.0000" in row["detail"] + + +def test_an_eligibility_loss_in_the_same_household_cannot_hide_a_double_count(): + """One household's ben_e contains ben_c and it loses £100 of ben_f at once.""" + formulas = model(interaction=0.0) + first = np.array([1.0, 0.0, 0.0]) + formulas["ben_e"] = lambda f: 50 + first * f("ben_c") + formulas["ben_f"] = lambda f: np.where( + (f("ben_c") > 100.5) & (first == 1), 0.0, 100.0 + ) + formulas["gov_spending"] = lambda f: ( + f("ben_c") + f("ben_d") + f("ben_e") + f("ben_f") + ) + _, result = run( + formulas, partition(["tax_a"], ["ben_c", "ben_d", "ben_e", "ben_f"]) + ) + row = by_variable(result)["ben_c"] + assert row["status"] == "fail" and "ben_e rises" in row["detail"] + + +def test_a_declared_read_may_respond_but_not_one_for_one(): + formulas = model(contains=False) + formulas["tax_a"] = lambda f: 0.1 * f("x") + 0.3 * f("tax_b") + reading = partition(["tax_a", "tax_b"], reads={"tax_a": ["tax_b"]}) + _, result = run(formulas, reading) + assert by_variable(result)["tax_b"]["status"] == "pass" + _, undeclared = run(formulas, partition(["tax_a", "tax_b"])) + assert by_variable(undeclared)["tax_b"]["status"] == "fail" + _, containing = run(model(contains=True), reading) + row = by_variable(containing)["tax_b"] + assert row["status"] == "fail" and "contains it" in row["detail"] + + +def test_supplied_inputs_are_restored_after_perturbation(): + formulas = model(contains=False) + formulas.pop("tax_b") + sim = FakeSim(formulas, {"x": (1, 2, 3), "tax_b": (7, 0, 9)}, (1, 1, 1)) + part = partition(["tax_a", "tax_b"]) + snapshot = snapshot_inputs(sim, part.component_variables()) + result = counted_once(sim, part, YEAR, 1e-6, snapshot) + assert np.array_equal(sim.inputs[("tax_b", str(YEAR))], [7, 0, 9]) + assert result["reset_reproduces_baseline"] + assert by_variable(result)["tax_b"]["units"] == 2 + + +def test_zero_components_are_perturbed_everywhere(): + formulas = model(contains=False) + formulas["tax_b"] = lambda f: 0 * f("x") + formulas["ben_c"] = lambda f: 0 * f("x") + _, result = run(formulas, partition(["tax_a", "tax_b"], ["ben_c", "ben_d"])) + rows = by_variable(result) + assert rows["tax_b"]["status"] == "pass" and rows["tax_b"]["units"] == 3 + assert rows["ben_c"]["status"] == "pass" and rows["ben_c"]["units"] == 3 + + +def test_a_formula_result_computed_earlier_is_not_frozen_as_an_input(): + """On real policyengine-core, a holder's known periods include results.""" + from policyengine_core.country_template import CountryTaxBenefitSystem + from policyengine_core.simulations import SimulationBuilder + + system = CountryTaxBenefitSystem() + sim = SimulationBuilder().build_from_dict( + system, {"persons": {"a": {"salary": {"2026-01": 3000}}}} + ) + sim.calculate("income_tax", "2026-01") + assert sim.get_holder("income_tax").get_known_periods() + snapshot = snapshot_inputs(sim, ["salary", "income_tax"]) + assert set(snapshot) == {"salary"} + + +@settings(max_examples=40, deadline=None) +@given( + x=st.lists(st.floats(1, 1e6), min_size=1, max_size=8), + weight=st.floats(0.1, 1e4), + contains=st.booleans(), +) +def test_the_model_coefficient_is_two_exactly_when_a_head_contains_another( + x, weight, contains +): + _, result = run( + model(contains=contains), + partition(["tax_a", "tax_b"]), + x=x, + weights=[weight] * len(x), + ) + row = by_variable(result)["tax_b"] + assert row["model_coefficients"]["gov_tax"] == pytest.approx( + 2 if contains else 1, rel=1e-6 + ) + assert (row["status"] == "fail") == contains + assert result["reset_reproduces_baseline"] diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py new file mode 100644 index 0000000000..2bbafc779a --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_measure.py @@ -0,0 +1,151 @@ +"""Measurement bookkeeping: check scopes, reform costing checks, digests.""" + +import dataclasses +from types import SimpleNamespace + +import numpy as np +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk.utils.aggregate_impact.api import to_report +from policyengine_uk.utils.aggregate_impact.measure import ( + Recorder, + _finite_metrics_check, + _digest_tool_files, + definitions_digest, + measure_reforms, + tool_digest, +) +from policyengine_uk.utils.aggregate_impact.reforms import load_reforms, parse_reforms +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +SPEC = load_spec() +YEARS = [2025, 2026] + + +class YearSim: + """A simulation whose gov_balance per household depends on the year.""" + + def __init__(self, balance_by_year): + self.balance = balance_by_year + entity = SimpleNamespace(key="household") + self.tax_benefit_system = SimpleNamespace( + variables={ + "gov_balance": SimpleNamespace(entity=entity), + "household_weight": SimpleNamespace(entity=entity), + } + ) + + def calculate(self, variable, period, map_to=None, unweighted=True): + if variable == "household_weight": + return np.ones(2) + return np.full(2, self.balance[int(period)] / 2) + + +def test_checks_remember_which_years_failed(): + rec = Recorder() + rec.scope = "2025" + rec.check("identity:x", "x", True) + rec.scope = "2026" + rec.check("identity:x", "x", False, "broken") + record = rec.checks["identity:x"] + assert record["status"] == "fail" and record["failing"] == {"2026": "broken"} + + +def test_a_check_that_did_not_apply_is_skipped_not_passed(): + rec = Recorder() + rec.check("counted_once:x", "x", None, "zero for every unit") + assert rec.checks["counted_once:x"]["status"] == "skipped" + rec.check("counted_once:x", "x", False, "broken") + assert rec.checks["counted_once:x"]["status"] == "fail" + + +def test_a_rebuilt_baseline_must_match_in_every_year(): + rec = Recorder() + baseline = {2025: 10.0, 2026: 20.0} + drifting = YearSim({2025: 10.0, 2026: 20.5}) + measure_reforms(lambda reform=None: drifting, None, baseline, {}, [], YEARS, rec) + record = rec.checks["determinism:rebuilt_baseline"] + assert record["status"] == "fail" and list(record["failing"]) == ["2026"] + + rec = Recorder() + measure_reforms( + lambda reform=None: YearSim(baseline), None, baseline, {}, [], YEARS, rec + ) + assert rec.checks["determinism:rebuilt_baseline"]["status"] == "pass" + + +def test_a_non_finite_reform_costing_fails_the_finiteness_check(): + rec = Recorder() + baseline = {2025: 10.0, 2026: 20.0} + reform = parse_reforms({"reforms": [{"name": "Break it", "parameters": {"a": 1}}]}) + + def factory(reform=None): + return YearSim({2025: 11.0, 2026: float("nan")} if reform else baseline) + + measure_reforms(factory, None, baseline, {}, reform, YEARS, rec) + assert rec.checks.get("reform_runs:break_it") is None # recorded by measure() + _finite_metrics_check(rec) + record = rec.checks["finite:metrics"] + assert record["status"] == "fail" and "reforms.break_it" in record["details"][0] + + +def test_the_definitions_digest_follows_what_is_measured(): + reforms = load_reforms() + digest = definitions_digest(SPEC, reforms) + assert digest == definitions_digest(load_spec(), load_reforms()) + shorter = dataclasses.replace(SPEC, last_year=SPEC.last_year - 1) + assert definitions_digest(shorter, reforms) != digest + assert definitions_digest(SPEC, reforms[:-1]) != digest + other_measures = dataclasses.replace(SPEC, raw={**SPEC.raw, "poverty": {}}) + assert definitions_digest(other_measures, reforms) != digest + + +@settings(max_examples=50, deadline=None) +@given(first=st.integers(2020, 2030), length=st.integers(1, 8), drop=st.integers(0, 9)) +def test_the_digest_separates_different_horizons_and_reform_sets(first, length, drop): + reforms = load_reforms() + a = dataclasses.replace(SPEC, first_year=first, last_year=first + length - 1) + b = dataclasses.replace(SPEC, first_year=first, last_year=first + length) + assert definitions_digest(a, reforms) != definitions_digest(b, reforms) + fewer = reforms[:drop] + reforms[drop + 1 :] + assert definitions_digest(a, fewer) != definitions_digest(a, reforms) + + +def test_the_long_format_report_carries_each_checks_failing_years(): + rec = Recorder() + rec.scope = "2025" + rec.check("identity:x", "x", False, "broken in 2025") + rec.scope = "2026" + rec.check("identity:x", "x", False, "broken in 2026") + rec.scope = "" + rec.check("closure", "closure", True) + rec.check("counted_once:y", "y", None, "zero for every unit") + rec.metric("receipts.income_tax", "Income tax", "receipts", "gbp", 2025, 1.0) + report = to_report( + { + "status": "ok", + "years": [2025, 2026], + "provenance": {"metrics_version": "v"}, + "partition": {"sha256": "p"}, + "metrics": list(rec.metrics.values()), + "checks": list(rec.checks.values()), + } + ) + rows = {row["identity_id"]: row for row in report["identities"]} + assert rows["identity:x"]["failing_scopes"] == ["2025", "2026"] + assert rows["identity:x"]["status"] == "fail" + assert rows["closure"]["failing_scopes"] == [] + assert rows["counted_once:y"]["status"] == "skipped" + assert rows["counted_once:y"]["failing_scopes"] == [] + metric = report["metrics"][0] + assert ( + metric["metric_id"] == "receipts.income_tax" + and metric["partition"] == "income_tax" + ) + assert metric["kind"] == "receipt" and metric["year"] == 2025 + + +def test_the_tool_digest_is_taken_at_load_and_matches_the_files(): + assert tool_digest() == _digest_tool_files() + assert len(tool_digest()) == 64 diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py new file mode 100644 index 0000000000..ab932f37d3 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_partition.py @@ -0,0 +1,203 @@ +"""The fiscal partition against the model's own component lists. + +These run without data. The partition was written from the receipts +classification and each variable's formula; closure compares it with +GOV_TAX_VARIABLES and its siblings, which are a second, independent statement +of the same totals. +""" + +import copy + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem +from policyengine_uk.utils.aggregate_impact.partition import ( + PartitionError, + closure_problems, + default_partition_path, + group_label, + load_partition, + parse_partition, + read_model_lists, + structural_problems, +) + +PATH = default_partition_path() + + +@pytest.fixture(scope="module") +def system(): + return CountryTaxBenefitSystem() + + +@pytest.fixture(scope="module") +def shipped(): + return load_partition(PATH) + + +@pytest.fixture(scope="module") +def raw(): + with open(PATH) as f: + return yaml.safe_load(f) + + +@pytest.fixture(scope="module") +def model_lists(shipped): + lists, problems = read_model_lists(shipped) + assert not problems + return lists + + +def test_shipped_partition_is_found_and_loads(shipped): + assert PATH is not None and PATH.name == "fiscal_partition.yaml" + assert shipped.schema_version == 1 + assert shipped.tax_heads and shipped.spending_heads + + +def test_every_model_total_is_exactly_its_heads_and_declared_double_counts( + shipped, model_lists, system +): + assert set(model_lists) == { + "gov_tax", + "household_tax", + "pre_budget_change_household_tax", + "gov_spending", + "household_benefits", + } + assert closure_problems(shipped, model_lists, system.variables) == [] + + +def test_every_head_is_a_money_variable_with_its_declared_entity(shipped, system): + for head in shipped.heads: + variable = system.variables[head.variable] + assert variable.value_type is float, head.id + assert variable.definition_period in ("year", "month"), head.id + assert head.entity == variable.entity.key, head.id + + +def test_receipts_and_spending_are_separate(shipped): + for head in shipped.tax_heads: + assert not set(head.totals) & {"gov_spending", "household_benefits"} + for head in shipped.spending_heads: + assert not set(head.totals) & {"gov_tax", "household_tax"} + + +def test_dropped_when_matches_the_council_tax_abolition_formulas(shipped): + switch = "gov.contrib.abolish_council_tax" + assert shipped.dropped("gov_tax", [switch]) == { + "council_tax", + "high_value_council_tax_surcharge", + } + assert shipped.dropped("gov_spending", [switch]) == {"council_tax_benefit"} + assert shipped.dropped("gov_tax") == set() + assert set(shipped.components("gov_tax")) - set( + shipped.components("gov_tax", [switch]) + ) == {"council_tax", "high_value_council_tax_surcharge"} + + +def test_group_labels_read_as_sentence_case(): + assert group_label("excise_duties") == "Excise duties" + assert group_label("Income tax and NICs") == "Income tax and NICs" + + +def _remove_head(raw, head_id): + """The partition without one head and every reference to it.""" + out = copy.deepcopy(raw) + removed = next( + h for h in raw["tax_heads"] + raw["spending_heads"] if h["id"] == head_id + ) + for key in ("tax_heads", "spending_heads"): + out[key] = [h for h in out[key] if h["id"] != head_id] + for head in out[key]: + if head.get("reads"): + head["reads"] = [r for r in head["reads"] if r.get("head") != head_id] + out["exclusions"] = [ + e + for e in out.get("exclusions") or [] + if e.get("head") != head_id and head_id not in (e.get("spans") or []) + ] + out["model_double_counts"] = [ + d + for d in out.get("model_double_counts") or [] + if d["contained_in"] != removed["variable"] + ] + for spec in (out.get("totals") or {}).values(): + for parameter, variables in ((spec or {}).get("dropped_when") or {}).items(): + spec["dropped_when"][parameter] = [ + v for v in variables if v != removed["variable"] + ] + return out + + +def _mutations(raw): + """Each yields a partition dict that a check must reject, and which check.""" + out = [] + a = copy.deepcopy(raw) + a["tax_heads"][1]["variable"] = a["tax_heads"][0]["variable"] + out.append(("duplicate head variable", a, "structural")) + out.append( + ("missing head", _remove_head(raw, raw["tax_heads"][0]["id"]), "closure") + ) + c = copy.deepcopy(raw) + c["spending_heads"].append( + { + "id": "not_a_model_component", + "label": "x", + "variable": "employment_income", + "totals": ["gov_spending"], + } + ) + out.append(("extra component", c, "closure")) + d = copy.deepcopy(raw) + if d.get("model_double_counts"): + d["model_double_counts"][0]["contained_in"] = "vat" + out.append(("double count in a total its container is not in", d, "structural")) + e = copy.deepcopy(raw) + e["tax_heads"][0]["totals"] = list(e["tax_heads"][0]["totals"]) + ["gov_spending"] + out.append(("tax head in a spending total", e, "structural")) + f = copy.deepcopy(raw) + f["tax_heads"][0]["id"] = f["tax_heads"][1]["id"] + out.append(("duplicate head id", f, "structural")) + return out + + +def test_each_kind_of_broken_partition_is_rejected(raw, model_lists, system): + for name, mutated, check in _mutations(raw): + if check == "structural": + with pytest.raises(PartitionError): + parse_partition(mutated) + else: + partition = parse_partition(mutated) + assert closure_problems(partition, model_lists, system.variables), name + + +@settings(max_examples=30, deadline=None) +@given(data=st.data()) +def test_component_sets_do_not_depend_on_head_order(raw, data): + shuffled = copy.deepcopy(raw) + shuffled["tax_heads"] = data.draw(st.permutations(shuffled["tax_heads"])) + shuffled["spending_heads"] = data.draw(st.permutations(shuffled["spending_heads"])) + a, b = parse_partition(raw), parse_partition(shuffled) + for total in a.totals: + assert sorted(a.components(total)) == sorted(b.components(total)) + + +@settings(max_examples=30, deadline=None) +@given(data=st.data()) +def test_removing_any_head_breaks_closure_of_each_total_it_was_in( + raw, model_lists, data +): + heads = raw["tax_heads"] + raw["spending_heads"] + removed = heads[data.draw(st.integers(0, len(heads) - 1))] + problems = closure_problems( + parse_partition(_remove_head(raw, removed["id"])), model_lists + ) + for total in removed["totals"]: + assert any(total in p and removed["variable"] in p for p in problems) + + +def test_structural_checks_pass_on_the_shipped_file(shipped): + assert structural_problems(shipped) == [] diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py new file mode 100644 index 0000000000..ccccd7cec8 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_reforms.py @@ -0,0 +1,142 @@ +"""The standard reform set and the responsiveness rule.""" + +from pathlib import Path + +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem +from policyengine_uk.utils.aggregate_impact.reforms import ( + ReformSpecError, + Responds, + load_reforms, + parameter_node, + parse_reforms, + reform_parameters, + responsiveness, +) +from policyengine_uk.utils.aggregate_impact.spec import load_spec + +TESTS = Path(__file__).resolve().parents[1] +YEARS = load_spec().years +STANDARD = load_reforms() + + +@pytest.fixture(scope="module") +def system(): + return CountryTaxBenefitSystem() + + +def test_every_standard_reform_says_what_it_must_move(system): + assert len(STANDARD) == 10 + assert len({r.id for r in STANDARD}) == len(STANDARD) + for reform in STANDARD: + assert reform.responds is not None, reform.id + assert reform.responds.variable in system.variables, reform.id + assert reform.responds.minimum > 0 + assert bool(reform.parameters) != bool(reform.scale), reform.id + + +def test_standard_set_contains_the_reform_cost_fixtures_unchanged(): + with open(TESTS / "microsimulation" / "reforms_config.yaml") as f: + fixtures = yaml.safe_load(f)["reforms"] + by_name = {r.name: r for r in STANDARD} + for fixture in fixtures: + assert dict(by_name[fixture["name"]].parameters) == fixture["parameters"] + + +def test_every_named_parameter_exists(system): + for reform in STANDARD: + for path in list(reform.parameters) + list(reform.scale): + assert parameter_node(system.parameters, path) is not None, path + + +def test_bracket_paths_resolve(system): + node = parameter_node(system.parameters, "gov.hmrc.income_tax.rates.uk[0].rate") + assert node("2026-01-01") == pytest.approx(0.2) + + +def test_scaled_reforms_are_the_baseline_times_the_factor_in_every_year(system): + for reform in STANDARD: + changes = reform_parameters(reform, system.parameters, YEARS) + for path, factor in reform.scale.items(): + assert set(changes[path]) == {str(y) for y in YEARS} + node = parameter_node(system.parameters, path) + for year in YEARS: + assert changes[path][str(year)] == pytest.approx( + node(f"{year}-01-01") * factor + ) + for path, value in reform.parameters.items(): + assert changes[path] == value + + +@pytest.mark.parametrize( + "entry", + [ + {"name": "no change"}, + { + "name": "x", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "sideways", "minimum": 1}, + }, + { + "name": "x", + "parameters": {"a": 1}, + "responds": {"variable": "v", "direction": "up", "minimum": 0}, + }, + {"name": "x", "scale": {"a": 0}}, + ], +) +def test_malformed_reforms_are_rejected(entry): + with pytest.raises(ReformSpecError): + parse_reforms({"reforms": [entry]}) + + +def test_duplicate_ids_are_rejected(): + entry = {"name": "Same name", "parameters": {"a": 1}} + with pytest.raises(ReformSpecError): + parse_reforms({"reforms": [entry, dict(entry)]}) + + +def test_fixture_schema_parses_without_responds(): + reforms = parse_reforms({"reforms": [{"name": "Raise X", "parameters": {"a": 1}}]}) + assert reforms[0].id == "raise_x" and reforms[0].responds is None + + +changes = st.dictionaries( + st.integers(2025, 2035), + st.floats(-1e11, 1e11, allow_nan=False), + min_size=1, + max_size=8, +) + + +@settings(max_examples=200, deadline=None) +@given(changes=changes, minimum=st.floats(1, 1e10)) +def test_responsive_exactly_when_every_year_moves_enough_the_right_way( + changes, minimum +): + up, _ = responsiveness(Responds("v", "up", minimum), changes) + assert up == all(c >= minimum for c in changes.values()) + # Mirror image: the same reform read as a fall of the negated changes. + down, _ = responsiveness( + Responds("v", "down", minimum), {y: -c for y, c in changes.items()} + ) + assert down == up + + +@settings(max_examples=100, deadline=None) +@given(changes=changes, minimum=st.floats(1, 1e10), year=st.integers(2025, 2035)) +def test_one_inert_or_wrong_signed_year_fails(changes, minimum, year): + for bad in (0.0, -abs(minimum)): + ok, detail = responsiveness( + Responds("v", "up", minimum), {**changes, year: bad} + ) + assert not ok and str(year)[2:] in detail + + +def test_no_measured_year_fails(): + ok, detail = responsiveness(Responds("v", "up", 1.0), {}) + assert not ok and "no year" in detail diff --git a/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py new file mode 100644 index 0000000000..0409d9c1a1 --- /dev/null +++ b/policyengine_uk/tests/aggregate_impact/test_aggregate_impact_workflow.py @@ -0,0 +1,82 @@ +"""Data protection: only aggregates may leave the aggregate-impact job. + +The certified dataset is licensed survey microdata. The measurement keeps +household-level detail for the decile comparison under ``private/``; these +tests fail if the workflow could publish it, or could put the dataset where +other workflows can read it. +""" + +import json +from pathlib import Path + +import numpy as np +import yaml + +from policyengine_uk.utils.aggregate_impact.measure import ( + HOUSEHOLDS_FILE, + PRIVATE_DIRECTORY, + read_result, + write_result, +) + +REPO = Path(__file__).resolve().parents[3] +WORKFLOW = REPO / ".github" / "workflows" / "aggregate-impact.yaml" +ALLOWED_UPLOAD_PATHS = {"report"} + + +def steps(): + workflow = yaml.safe_load(WORKFLOW.read_text()) + return [step for job in workflow["jobs"].values() for step in job["steps"]] + + +def test_only_the_aggregate_report_is_uploaded(): + uploads = [s for s in steps() if "upload-artifact" in str(s.get("uses", ""))] + assert uploads, "the report should be uploaded" + for step in uploads: + paths = {p.strip() for p in str(step["with"]["path"]).splitlines() if p.strip()} + assert paths <= ALLOWED_UPLOAD_PATHS, paths + + +def test_the_dataset_is_never_cached(): + for step in steps(): + assert "actions/cache" not in str(step.get("uses", "")), step.get("name") + + +def test_nothing_private_is_copied_into_the_report(): + text = WORKFLOW.read_text() + copies = [ + line.strip() for line in text.splitlines() if line.strip().startswith("cp ") + ] + for line in copies: + if "report/" in line: + assert line.split()[1].endswith("result.json"), line + assert f"{PRIVATE_DIRECTORY}/" not in "\n".join(copies) + assert HOUSEHOLDS_FILE not in "\n".join(copies) + + +def test_the_job_runs_both_sides_itself(): + workflow = yaml.safe_load(WORKFLOW.read_text()) + assert list(workflow["jobs"]) == ["report"], ( + "base and head must be measured in one job, so that household detail " + "never has to travel between jobs as an artifact" + ) + assert not [s for s in steps() if "download-artifact" in str(s.get("uses", ""))] + + +def test_household_detail_is_written_only_under_private(tmp_path): + households = { + "2026": { + "household_id": np.arange(5), + "household_weight": np.ones(5), + "household_count_people": np.ones(5), + "decile": np.arange(5), + "income": np.linspace(1, 2, 5), + } + } + write_result(tmp_path, {"status": "ok", "metrics": []}, households) + top_level = sorted(p.name for p in tmp_path.iterdir()) + assert top_level == [PRIVATE_DIRECTORY, "result.json"] + assert (tmp_path / PRIVATE_DIRECTORY / HOUSEHOLDS_FILE).exists() + result, loaded = read_result(tmp_path) + assert json.loads((tmp_path / "result.json").read_text()) == result + assert np.array_equal(loaded["2026"]["income"], households["2026"]["income"]) diff --git a/policyengine_uk/tests/certified_dataset.yaml b/policyengine_uk/tests/certified_dataset.yaml new file mode 100644 index 0000000000..994f08b5ba --- /dev/null +++ b/policyengine_uk/tests/certified_dataset.yaml @@ -0,0 +1,20 @@ +# The dataset that credentialed test jobs and the aggregate-impact report run +# on. It is pinned by Hugging Face commit, not by tag, and every run checks the +# file's SHA-256 against `sha256`, so the selection cannot move underneath a +# test. Change it only to another certified pair, together with the reform +# regression fixtures in microsimulation/reforms_config.yaml, which record the +# same dataset. +data_package: policyengine-uk-data +data_version: 1.58.0 +repo_id: policyengine/policyengine-uk-data-private +path: enhanced_frs_2024_25.h5 +# Commit the 1.58.0 tag pointed to on 2026-10-11. +revision: 47bbb58d3f6467aa4a7059bc6110d2bdcf699b29 +hf_tag: 1.58.0 +sha256: 5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6 +size_bytes: 128369110 +certification: + certified_by: policyengine.py bundle uk-6.2.6 (PolicyEngine/policyengine.py#555) + certified_for_model_version: 2.125.1 + built_with_model_version: 2.122.2 + release_manifest_revision: fdeba38517bd86067f34c0a5d8ed8923efb3b10a diff --git a/policyengine_uk/tests/conftest.py b/policyengine_uk/tests/conftest.py index d611a0c1c5..5fe86e974c 100644 --- a/policyengine_uk/tests/conftest.py +++ b/policyengine_uk/tests/conftest.py @@ -2,18 +2,162 @@ import os import re import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Optional import pytest from policyengine_core.taxbenefitsystems import TaxBenefitSystem -DEFAULT_TEST_DATASET_URL = ( - "hf://policyengine/policyengine-uk-data-private/enhanced_frs_2023_24.h5@1.40.3" -) +from policyengine_uk.utils.aggregate_impact.dataset import load_pin, materialise -if os.environ.get("HUGGING_FACE_TOKEN") and not os.environ.get( - "POLICYENGINE_UK_DEFAULT_DATASET" -): - os.environ["POLICYENGINE_UK_DEFAULT_DATASET"] = DEFAULT_TEST_DATASET_URL +# Microsimulation tests run on one certified dataset, pinned by commit and +# SHA-256 in certified_dataset.yaml. +CERTIFIED_DATASET_PIN = Path(__file__).with_name("certified_dataset.yaml") +CERTIFIED_DATASET = load_pin(CERTIFIED_DATASET_PIN) +DEFAULT_TEST_DATASET_URL = CERTIFIED_DATASET.uri + +DATASET_ENV_VAR = "POLICYENGINE_UK_DEFAULT_DATASET" +TOKEN_ENV_VAR = "HUGGING_FACE_TOKEN" +# Mandatory-data mode, for CI jobs that must run the microsimulation tests: +# with it set, a test that needs data fails when the certified dataset is +# unavailable, instead of being skipped, and runs only on that dataset. +REQUIRE_DATA_ENV_VAR = "POLICYENGINE_UK_REQUIRE_TEST_DATA" +_TRUE = {"1", "true", "yes", "on"} +_FALSE = {"", "0", "false", "no", "off"} + + +@dataclass(frozen=True) +class TestDataPlan: + """What the microsimulation tests run on, decided from the environment.""" + + __test__ = False # not a test class + + action: str # "run", "skip" or "fail" + dataset: Optional[str] + pinned: bool + mandatory: bool + reason: str + + +def plan_test_data(env: Mapping[str, str], pinned_uri: str) -> TestDataPlan: + """Decide how microsimulation tests get their data. + + - An explicit ``POLICYENGINE_UK_DEFAULT_DATASET`` wins; in mandatory mode + it must be the pinned certified dataset. + - Otherwise a Hugging Face token selects the pinned certified dataset. + - Otherwise the tests skip, or in mandatory mode fail. + + An unrecognised mandatory-mode value fails closed. + """ + flag = str(env.get(REQUIRE_DATA_ENV_VAR, "")).strip().lower() + explicit = env.get(DATASET_ENV_VAR) or None + token = env.get(TOKEN_ENV_VAR) or None + if flag not in _TRUE | _FALSE: + return TestDataPlan( + "fail", + explicit, + False, + True, + f"{REQUIRE_DATA_ENV_VAR}={env.get(REQUIRE_DATA_ENV_VAR)!r} is not a " + "recognised true or false value", + ) + mandatory = flag in _TRUE + if explicit: + pinned = explicit == pinned_uri + if mandatory and not pinned: + return TestDataPlan( + "fail", + explicit, + False, + True, + f"mandatory-data mode runs only on the pinned certified dataset " + f"{pinned_uri}, not {explicit}", + ) + return TestDataPlan( + "run", + explicit, + pinned, + mandatory, + "pinned certified dataset" if pinned else "dataset set explicitly", + ) + if token: + return TestDataPlan( + "run", pinned_uri, True, mandatory, "pinned certified dataset" + ) + if mandatory: + return TestDataPlan( + "fail", + None, + False, + True, + f"mandatory-data mode needs {TOKEN_ENV_VAR} (or {DATASET_ENV_VAR} set " + "to the pinned certified dataset)", + ) + return TestDataPlan( + "skip", + None, + False, + False, + f"Requires {DATASET_ENV_VAR} or {TOKEN_ENV_VAR} for microsimulation " + "dataset access", + ) + + +TEST_DATA_PLAN = plan_test_data(os.environ, DEFAULT_TEST_DATASET_URL) + +if TEST_DATA_PLAN.action == "run" and not os.environ.get(DATASET_ENV_VAR): + os.environ[DATASET_ENV_VAR] = TEST_DATA_PLAN.dataset + + +_verification: dict = {} + + +def verify_certified_dataset() -> Optional[str]: + """Fetch the pinned dataset once per process and check its SHA-256. + + Returns None when the file matches, or why it could not be verified. + """ + if "problem" not in _verification: + try: + materialise(CERTIFIED_DATASET) + _verification["problem"] = None + except Exception as error: # noqa: BLE001 - every cause fails the tests + _verification["problem"] = ( + f"The pinned certified dataset {CERTIFIED_DATASET.describe()} could " + f"not be fetched and verified: {type(error).__name__}: {error}" + ) + return _verification["problem"] + + +# Test files that read the unpinned latest dataset on purpose, to notice when +# a new data release breaks the model. They are data tests (they need +# credentials, and fail without them in mandatory mode), but what they run on +# is by design not the pinned certified dataset. +UNPINNED_BY_DESIGN = ("test_latest_data_smoke.py",) + + +def needs_data(item) -> bool: + """Whether a test carries the microsimulation marker. + + The marker, not the keyword: keywords include directory names, which + would sweep in the data-free tests under ``microsimulation/``. + """ + return item.get_closest_marker("microsimulation") is not None + + +def runs_on_unpinned_data(item) -> bool: + return item.path.name in UNPINNED_BY_DESIGN + + +def pytest_report_header(config): + plan = TEST_DATA_PLAN + mode = "mandatory" if plan.mandatory else "optional" + target = CERTIFIED_DATASET.describe() if plan.pinned else (plan.dataset or "none") + return [ + f"microsimulation data ({mode}): {plan.action} on {target} ({plan.reason})", + "unpinned by design: " + ", ".join(UNPINNED_BY_DESIGN), + ] @pytest.fixture(scope="module") @@ -30,19 +174,37 @@ def cloned_uk_tax_benefit_system(): def pytest_collection_modifyitems(config, items): - has_default_dataset = bool(os.environ.get("POLICYENGINE_UK_DEFAULT_DATASET")) - if has_default_dataset: + """Without data, microsimulation tests skip, unless data is mandatory.""" + if TEST_DATA_PLAN.action != "skip": return + skip = pytest.mark.skip(reason=TEST_DATA_PLAN.reason) + for item in items: + if needs_data(item): + item.add_marker(skip) + + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + """Fail data tests in mandatory mode without data; verify pinned bytes. - skip_microsimulation = pytest.mark.skip( - reason=( - "Requires POLICYENGINE_UK_DEFAULT_DATASET or HUGGING_FACE_TOKEN " - "for microsimulation dataset access" + A failure here is reported as an error in the test's setup, so the run + fails rather than passing with the test unexecuted. The hook runs before + pytest's own skip handling, so a skip marker on a data test cannot turn + missing data into a skip. + """ + if not needs_data(item): + return + plan = TEST_DATA_PLAN + if plan.action == "fail": + pytest.fail( + f"Mandatory-data mode: {plan.reason}. This test needs the certified " + "dataset and must not be skipped.", + pytrace=False, ) - ) - for item in items: - if "microsimulation" in item.keywords: - item.add_marker(skip_microsimulation) + if plan.action == "run" and plan.pinned and not runs_on_unpinned_data(item): + problem = verify_certified_dataset() + if problem: + pytest.fail(problem, pytrace=False) # policyengine-core loads every variable file of each new tax-benefit system as diff --git a/policyengine_uk/tests/microsimulation/reforms_config.yaml b/policyengine_uk/tests/microsimulation/reforms_config.yaml index 57bcc7d2ef..c48a59ac45 100644 --- a/policyengine_uk/tests/microsimulation/reforms_config.yaml +++ b/policyengine_uk/tests/microsimulation/reforms_config.yaml @@ -2,38 +2,135 @@ # These expected impacts are derived from current model output and are used to # catch unintended behavior changes. They are not external fiscal estimates # unless a reform entry is explicitly annotated otherwise. +# +# Each impact is the change in weighted gov_balance in 2029, in £bn, on the +# dataset below, which must be the one tests/certified_dataset.yaml pins. +# `receipts` keeps every earlier value with the model and data that produced +# it, so a change of dataset and a change of model stay separable. Tests ran +# on whatever enhanced FRS was latest until 2026-04-05, then on uk-data 1.40.3 +# (enhanced_frs_2023_24), and from 2026-10-11 on the certified 1.58.0 pair. The +# 2026-10-11 receipts on 1.40.3 show how much of each move is the data rather +# than the model. +dataset: + data_version: 1.58.0 + path: enhanced_frs_2024_25.h5 + revision: 47bbb58d3f6467aa4a7059bc6110d2bdcf699b29 + sha256: 5ba399ea5ab9a6c7d4e9aaf6c623080beb34d9ad5f330f6244dbad72b8d960b6 reforms: - name: Raise basic rate by 1pp - expected_impact: 7.8 + expected_impact: 7.1 parameters: gov.hmrc.income_tax.rates.uk[0].rate: 0.21 + receipts: + - impact: 7.8 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.34.5 (a1832dc77, #1205, 2025-07-11) + - impact: 7.346 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 7.132 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise higher rate by 1pp - expected_impact: 4.8 + expected_impact: 4.6 parameters: gov.hmrc.income_tax.rates.uk[1].rate: 0.42 + receipts: + - impact: 4.8 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.38.1 (fa6eb613e, #1231, 2025-07-15) + - impact: 3.961 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 4.609 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise personal allowance by ~800GBP/year - expected_impact: -4.2 + expected_impact: -4.1 parameters: gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 + receipts: + - impact: -4.2 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.59.0 (53c38d229, #1396, 2025-11-27) + - impact: -4.305 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -4.135 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise child benefit by 25GBP/week per additional child - expected_impact: -1.2 + expected_impact: -1.4 parameters: gov.hmrc.child_benefit.amount.additional: 25 + receipts: + - impact: -1.2 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.38.0 (bda6d2ddf, #1228, 2025-07-14) + - impact: -1.222 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -1.428 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Reduce Universal Credit taper rate to 20% - expected_impact: -18.1 + expected_impact: -21.3 tolerance: 3.0 parameters: gov.dwp.universal_credit.means_test.reduction_rate: 0.2 + receipts: + - impact: -18.1 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.78.0 (e4be118a7, #1572, 2026-04-11) + - impact: -18.354 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: -21.266 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise Class 1 main employee NICs rate to 10% - expected_impact: 13.0 + expected_impact: 12.6 parameters: gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 + receipts: + - impact: 13.0 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.59.0 (53c38d229, #1396, 2025-11-27) + - impact: 13.263 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 12.566 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise VAT standard rate by 2pp - expected_impact: 21.3 + expected_impact: 34.5 tolerance: 1.5 parameters: gov.hmrc.vat.standard_rate: 0.22 + # The move from 21.3 is the data: on 1.58.0 recorded full-rate consumption + # is 62% higher than on 1.40.3, and the VAT coverage factor (#1996, #2187) + # was set on older data. + receipts: + - impact: 21.3 + data: enhanced_frs_2023_24, unpinned (latest at the time) + model: 2.75.3 (bc59ba950, #1540, 2026-03-29) + - impact: 21.222 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 34.478 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) - name: Raise additional rate by 3pp - expected_impact: 4.4 + expected_impact: 3.8 parameters: gov.hmrc.income_tax.rates.uk[2].rate: 0.48 + receipts: + - impact: 4.4 + data: enhanced_frs_2022_23, unpinned (latest at the time) + model: 2.35.0 (88bdf190b, #1207, 2025-07-11) + - impact: 3.993 + data: 1.40.3 enhanced_frs_2023_24 + model: 2.126.8 (6d89338ab, 2026-10-11) + - impact: 3.756 + data: 1.58.0 enhanced_frs_2024_25 + model: 2.126.8 (6d89338ab, 2026-10-11) diff --git a/policyengine_uk/utils/aggregate_impact/README.md b/policyengine_uk/utils/aggregate_impact/README.md new file mode 100644 index 0000000000..051597e470 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/README.md @@ -0,0 +1,127 @@ +# Aggregate impact report + +This tool shows what a change does to national numbers. It runs the model on +the certified dataset for two checkouts (a pull request's base and its head) +and compares them. Each side runs every year from 2025-26 to 2031-32 as a full +microsimulation. + +The report covers: +- receipts by tax head; +- spending and caseloads by benefit; +- poverty; +- inequality; +- deciles; +- the standard reform costings. + +The `Aggregate impact` workflow (`.github/workflows/aggregate-impact.yaml`) +runs it on every pull request and keeps one comment on the PR up to date. + +## Data protection + +The certified dataset is licensed survey microdata. Only aggregates may be +published: `result.json`, the comment and the comparison summary. `measure` +also writes household-level detail for the decile comparison to +`private/households.npz`. Never upload, attach, commit or share anything +under `private/`, and never put the dataset in a shared cache. The workflow +measures both sides in one job so that this detail stays on the runner, and +`test_aggregate_impact_workflow.py` fails if the workflow could publish it. +Check details give counts and weighted totals, never one household's value. + +## Running it + +```bash +export HUGGING_FACE_TOKEN=... # read access to policyengine-uk-data-private +uv run python -m policyengine_uk.utils.aggregate_impact measure --label head \ + --pin policyengine_uk/tests/certified_dataset.yaml --out out/head +# ... the same for a base checkout, into out/base, then: +uv run python -m policyengine_uk.utils.aggregate_impact compare \ + --base out/base --head out/head --markdown comment.md --summary summary.json +``` + +`measure` downloads the pinned file if it isn't cached, checks its SHA-256, and +writes `result.json` (figures, checks, provenance) and +`private/households.npz` (household detail for the decile comparison). If it crashes, it still writes a +`result.json` recording the crash, so the comparison can report it. + +To measure an older checkout with this version of the tool, copy this +directory under another name and run it in that checkout's environment, as CI +does. The tool imports only its own modules and `policyengine_uk`'s public +`Microsimulation`: + +```bash +cp -R policyengine_uk/utils/aggregate_impact /tmp/tool/pe_uk_aggregate_impact +cd ../base-checkout && PYTHONPATH=/tmp/tool .venv/bin/python -m pe_uk_aggregate_impact measure ... +``` + +For use from other code (qualification in policyengine.py), see `api.py`: +`compute_budget_metrics(sim, years=...)` returns a long-format report. + +## What fails and what is flagged + +The comparison **fails** in these cases: +- the head crashes; +- base and head ran on different dataset bytes; +- a figure is not finite; +- the head breaks a check that the base passes. + +A check that fails on both sides is reported as pre-existing and does not +fail the job; `--strict` makes it fail. Moves above the thresholds in +`report_spec.yaml` are **flagged** for review, never failed, because intended +policy changes move numbers. + +The checks are: +- **closure:** for each model total (`gov_tax`, `household_tax`, + `pre_budget_change_household_tax`, `gov_spending`, `household_benefits`), the + components in the model's own list are exactly the partition's heads and + declared double counts; +- **identities**, checked household by household: + - the heads and declared double counts add up to each total; + - `gov_balance` equals `gov_tax` minus `gov_spending`; + - household net income equals market income plus benefits, less taxes and + pension contributions; +- **counted once:** each component is raised by £1 per unit. Each tax total it + is in must rise by exactly £1 per unit, and each spending total by at most + £1, because means tests can claw the rise back but nothing should add to + it. A declared double count must still be contained in its head; +- **signs, bounds and finiteness**, and **determinism:** a rebuilt + simulation reproduces the baseline exactly; +- **inert heads:** a head that is zero for every unit is a warning, unless the + partition marks it `zero_at_baseline: expected`; +- **reform responsiveness:** each standard reform must move the variable it + targets in the stated direction, by at least its stated minimum, in every + year. + +## The standard reforms + +`standard_reforms.yaml` lists the reforms costed on each side: the eight +reform-cost fixtures, plus a 10% rise in the Universal Credit standard +allowance and a 10% rise in the State Pension. Each says what it must move +(`responds`). A reform that the model ignores, or that moves its target the +wrong way, fails `responsive:` instead of passing as a small number. + +A relative reform (`scale`) is built from each side's own baseline: every +listed parameter gets that model year's baseline value times the factor, for +each year of the horizon. Pass `--reforms` to cost a different file, or +`--no-reforms` to cost none. + +## The fiscal partition + +`policyengine_uk/data/fiscal_partition.yaml` divides each total into heads, +each measured by one variable. Its header defines the fields. Each side of a +comparison reads its own partition, so a change to the model's totals and the +matching change to the partition go in the same pull request. + +When you add a component to one of the model's lists, add a head for it, or +the closure check fails. When the model stops counting something twice, +remove its `model_double_counts` entry and give it a head, or the +counted-once check fails as stale. + +## Years + +Model year 2026 is fiscal year 2026-27. The dataset is extended explicitly to +2030 (`extend_single_year_dataset`). Values for 2031-32 come from +policyengine-core instead: +- inputs with an `uprating` index are uprated from their 2030 values; +- other inputs are carried over unchanged. + +So treat 2031-32 figures as provisional. diff --git a/policyengine_uk/utils/aggregate_impact/__init__.py b/policyengine_uk/utils/aggregate_impact/__init__.py new file mode 100644 index 0000000000..e8c17a26e6 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/__init__.py @@ -0,0 +1,14 @@ +"""Base-versus-head aggregate impact report on a certified dataset. + +The report measures one model checkout at a time (``measure``), on a dataset +whose bytes are verified against a pinned SHA-256, and then compares two +measurements (``compare``). A pull request's CI measures its base and its head +on the same dataset and posts the comparison as a comment. + +This package must only import ``policyengine_uk`` through its public +simulation API, and must import its own modules relatively: CI runs the pull +request head's copy of it against the base checkout's model. +""" + +TOOL_SCHEMA = "policyengine-uk-aggregate-impact/1" +COMMENT_MARKER = "" diff --git a/policyengine_uk/utils/aggregate_impact/__main__.py b/policyengine_uk/utils/aggregate_impact/__main__.py new file mode 100644 index 0000000000..d0b75a5b8a --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/__main__.py @@ -0,0 +1,193 @@ +"""Command line: ``measure`` one checkout, then ``compare`` two measurements. + + python -m policyengine_uk.utils.aggregate_impact measure --label head \\ + --pin policyengine_uk/tests/certified_dataset.yaml --out out/head + python -m policyengine_uk.utils.aggregate_impact compare \\ + --base out/base --head out/head --markdown comment.md --summary summary.json + +``measure`` exits 1 if the measurement crashed, after writing a result that +records the crash; ``compare`` exits 1 when the change fails the report. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import traceback +from pathlib import Path + +from . import TOOL_SCHEMA + + +def _sha256(path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def cmd_measure(args) -> int: + from .dataset import load_pin, materialise, verify_file + from .measure import write_result + + provenance = { + "git_sha": args.git_sha, + "git_ref": args.git_ref, + "tool": str(Path(__file__).resolve().parent), + } + try: + from .measure import measure + from .partition import default_partition_path, load_partition + from .spec import DEFAULT_SPEC_PATH, load_spec + + spec_path = Path(args.spec) if args.spec else DEFAULT_SPEC_PATH + spec = load_spec(spec_path, years=args.years) + provenance["spec_sha256"] = _sha256(spec_path) + + pin = load_pin(args.pin) + if args.dataset_file: + dataset_path = Path(args.dataset_file) + verified = verify_file(dataset_path, pin.sha256) + else: + dataset_path = materialise(pin) + verified = pin.sha256 + provenance["dataset"] = {**pin.as_dict(), "verified_sha256": verified} + + if args.partition: + partition_path, partition_source = Path(args.partition), "given" + else: + partition_path = default_partition_path() + partition_source = "measured model" + if partition_path is None: + if not args.fallback_partition: + raise FileNotFoundError( + "the measured policyengine_uk ships no fiscal partition; " + "pass --fallback-partition" + ) + partition_path, partition_source = ( + Path(args.fallback_partition), + "fallback", + ) + partition = load_partition(partition_path) + provenance["partition_source"] = partition_source + + from .reforms import DEFAULT_REFORMS_PATH, load_reforms + + reforms = [] + if not args.no_reforms: + reforms_path = Path(args.reforms) if args.reforms else DEFAULT_REFORMS_PATH + reforms = load_reforms(reforms_path) + provenance["reforms_sha256"] = _sha256(reforms_path) + + from policyengine_uk import Microsimulation + + sim = Microsimulation(dataset=str(dataset_path)) + dataset_object = sim.dataset + + def factory(reform=None): + return Microsimulation(dataset=dataset_object, reform=reform or None) + + result, households = measure( + sim, + factory, + partition, + spec, + label=args.label, + reforms=reforms, + run_guard=not args.no_guard, + provenance=provenance, + ) + write_result(args.out, result, households) + print( + f"{args.label}: measured {len(result['metrics'])} figures in {result['timings']['total']}s" + ) + return 0 + except Exception: # noqa: BLE001 - recorded for the comparison, then reported + crash = { + "schema": TOOL_SCHEMA, + "status": "crashed", + "label": args.label, + "provenance": provenance, + "error": traceback.format_exc(), + } + write_result(args.out, crash) + print(crash["error"], file=sys.stderr) + return 1 + + +def cmd_compare(args) -> int: + from .compare import compare + from .measure import read_result + from .render import render + from .spec import DEFAULT_SPEC_PATH, load_spec + + spec = load_spec(Path(args.spec) if args.spec else DEFAULT_SPEC_PATH) + base, base_households = read_result(args.base) if args.base else (None, None) + head, head_households = read_result(args.head) + summary = compare( + base, + head, + spec, + base_households=base_households, + head_households=head_households, + strict=args.strict, + ) + markdown = render(summary, spec, context={"run_url": args.run_url}) + if args.markdown: + Path(args.markdown).write_text(markdown) + if args.summary: + with open(args.summary, "w") as f: + json.dump(summary, f, indent=1, allow_nan=True, default=str) + print(f"aggregate impact: {summary['status']}") + for failure in summary["failures"]: + print(f"FAIL: {failure}") + return 1 if summary["status"] == "fail" else 0 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + prog="python -m policyengine_uk.utils.aggregate_impact" + ) + sub = parser.add_subparsers(dest="command", required=True) + + m = sub.add_parser("measure", help="measure one model checkout") + m.add_argument("--label", required=True) + m.add_argument("--pin", required=True, help="dataset pin YAML") + m.add_argument("--dataset-file", help="local copy of the pinned file (verified)") + m.add_argument("--partition", help="fiscal partition YAML to use") + m.add_argument( + "--fallback-partition", + help="partition to use if the measured model ships none", + ) + m.add_argument("--spec", help="report spec YAML") + m.add_argument("--years", help="override the horizon, e.g. 2025-2031") + m.add_argument( + "--reforms", + help="reform YAML to cost instead of the standard set (standard_reforms.yaml)", + ) + m.add_argument("--no-reforms", action="store_true", help="cost no reforms") + m.add_argument("--git-sha", default="") + m.add_argument("--git-ref", default="") + m.add_argument( + "--no-guard", action="store_true", help="skip the counted-once check" + ) + m.add_argument("--out", required=True) + m.set_defaults(func=cmd_measure) + + c = sub.add_parser("compare", help="compare a base and a head measurement") + c.add_argument("--base") + c.add_argument("--head", required=True) + c.add_argument("--spec") + c.add_argument("--markdown") + c.add_argument("--summary") + c.add_argument("--run-url", default="") + c.add_argument( + "--strict", action="store_true", help="fail on checks failing on both sides" + ) + c.set_defaults(func=cmd_compare) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/policyengine_uk/utils/aggregate_impact/api.py b/policyengine_uk/utils/aggregate_impact/api.py new file mode 100644 index 0000000000..a22eeb11d2 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/api.py @@ -0,0 +1,154 @@ +"""In-process entry point for qualification (policyengine.py, workstream W4). + +``compute_budget_metrics`` measures one simulation and returns a long-format +report: one row per metric and year, plus the identity and partition checks. +Two pairs are comparable when their reports share ``metrics_version`` (the +same tool code, and the same effective spec, horizon and reforms), whatever +model each pair runs; each model's own +fiscal partition is read from the measured package, and its digest is +``partition_version``. + +To hold the definitions fixed across models, copy this package directory +from one policyengine-uk commit and import the copy under another name: it +imports only its own modules and policyengine_uk's public simulation API. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path +from typing import Callable, Optional, Sequence + +from . import TOOL_SCHEMA +from .measure import measure +from .partition import Partition, default_partition_path, load_partition +from .reforms import load_reforms +from .spec import DEFAULT_SPEC_PATH, Spec, load_spec + +KIND_BY_SECTION = { + "receipts": "receipt", + "spending": "spending", + "caseloads": "caseload", + "poverty": "poverty", + "poverty_count": "poverty_count", + "inequality": "inequality", + "deciles": "decile", + "balance": "balance", + "model_totals": "model_total", + "double_counts": "double_count", + "household_totals": "household_total", + "population": "population", + "reforms": "reform", + "reform_targets": "reform_target", +} +PARTITIONED_SECTIONS = {"receipts", "spending", "caseloads"} + + +def _spec_for_years(spec: Spec, years: Optional[Sequence[int]]) -> Spec: + if not years: + return spec + years = sorted(int(y) for y in years) + guard_year = spec.guard_year if spec.guard_year in years else years[0] + return dataclasses.replace( + spec, first_year=years[0], last_year=years[-1], guard_year=guard_year + ) + + +def to_report(result: dict) -> dict: + """Flatten a measurement into the long-format qualification report.""" + provenance = result.get("provenance") or {} + report = { + "schema": TOOL_SCHEMA + "/report", + "status": result.get("status"), + "metrics_version": provenance.get("metrics_version"), + "partition_version": (result.get("partition") or {}).get("sha256"), + "years": result.get("years"), + "metrics": [], + "identities": [], + } + if result.get("status") != "ok": + report["error"] = result.get("error") + return report + for metric in result["metrics"]: + section = metric["section"] + head_id = ( + metric["id"].split(".", 1)[1] if section in PARTITIONED_SECTIONS else None + ) + for year, value in metric["values"].items(): + report["metrics"].append( + { + "metric_id": metric["id"], + "year": int(year), + "value": value, + "unit": metric["kind"], + "kind": KIND_BY_SECTION.get(section, section), + "partition": head_id, + "label": metric["label"], + } + ) + for check in result["checks"]: + report["identities"].append( + { + "identity_id": check["id"], + "year": None, + "status": check["status"], + "severity": check["severity"], + # The years the check failed in ("" for a check with no year), + # so a consumer can tell a newly failing year from an old one. + "failing_scopes": sorted(check.get("failing") or {}), + "detail": "; ".join(check["details"]), + } + ) + return report + + +def compute_budget_metrics( + sim, + factory: Optional[Callable] = None, + *, + years: Optional[Sequence[int]] = None, + partition: Optional[Partition] = None, + spec: Optional[Spec] = None, + reforms: Optional[Sequence] = None, + run_guard: bool = True, + label: str = "candidate", + provenance: Optional[dict] = None, +) -> dict: + """Measure ``sim`` (a fresh ``Microsimulation``) and return the report. + + ``factory(reform=None)`` builds another simulation on the same data; by + default it rebuilds ``type(sim)`` from ``sim.dataset``. ``reforms`` + defaults to the standard set (``standard_reforms.yaml``), each costed and + checked for responsiveness; pass ``[]`` to cost none. A failed identity + has status ``fail``; ``warn`` rows (a head that is zero everywhere) are + observations, not failures. + """ + spec = _spec_for_years(spec or load_spec(DEFAULT_SPEC_PATH), years) + if partition is None: + path = default_partition_path() + if path is None: + raise FileNotFoundError( + "the measured policyengine_uk ships no fiscal partition" + ) + partition = load_partition(path) + if factory is None: + dataset = sim.dataset + simulation_class = type(sim) + + def factory(reform=None): + return simulation_class(dataset=dataset, reform=reform or None) + + if reforms is None: + reforms = load_reforms() + provenance = dict(provenance or {}) + result, _ = measure( + sim, + factory, + partition, + spec, + label=label, + reforms=reforms, + run_guard=run_guard, + provenance=provenance, + ) + return to_report(result) diff --git a/policyengine_uk/utils/aggregate_impact/compare.py b/policyengine_uk/utils/aggregate_impact/compare.py new file mode 100644 index 0000000000..ec288a4587 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/compare.py @@ -0,0 +1,486 @@ +"""Compare a base and a head measurement. + +The comparison fails when either side did not run, when they ran on +different data, when the head has a non-finite figure, when a check the base +ran is missing from the head, or when the head fails a check in a scope (a +year) where the base passes it. It reports, without failing, checks that +fail in the same scopes on both sides (pre-existing) and moves above the +flag thresholds. +""" + +from __future__ import annotations + +import math +from typing import Iterable, Mapping, Optional + +import numpy as np + +from . import TOOL_SCHEMA +from .spec import Spec, fiscal_year_label + +# Failing these on the head blocks the change whatever the base did: a +# non-finite or crashed result cannot be compared at all. +ALWAYS_BLOCKING_PREFIXES = ("finite:", "weights:") + +# Every head measurement must have run these. +REQUIRED_CHECKS = ( + "closure", + "finite:metrics", + "weights:person", + "weights:benunit", + "weights:household", +) + +FLAG_KIND_BY_SECTION = { + "reforms": "reform_gbp", + "reform_targets": "reform_gbp", +} + +DECILES = tuple(range(1, 11)) + + +def _metric_map(result: Optional[Mapping]) -> dict: + if not result or result.get("status") != "ok": + return {} + return {m["id"]: m for m in result.get("metrics", [])} + + +def _check_map(result: Optional[Mapping]) -> dict: + if not result or result.get("status") != "ok": + return {} + return {c["id"]: c for c in result.get("checks", [])} + + +def _finite(value) -> bool: + return ( + value is not None and isinstance(value, (int, float)) and math.isfinite(value) + ) + + +def classify_check( + base_status: Optional[str], + head_status: Optional[str], + blocking: bool, + base_failing: Iterable[str] = (), + head_failing: Iterable[str] = (), +) -> str: + """How a check's outcome on the head relates to the base. + + ``base_failing`` and ``head_failing`` are the scopes (years) each side + failed in. A failure is pre-existing only if the base fails in every + scope the head fails in; a newly failing year is a regression. + + Returns one of: ``pass``, ``regression``, ``pre_existing``, ``fixed``, + ``new_warning``, ``warning``, ``skipped``, ``removed``. + """ + if head_status is None: + return "removed" + if head_status == "skipped": + return "skipped" + if head_status == "pass": + return "fixed" if base_status in ("fail", "warn") else "pass" + if head_status == "warn": + return "warning" if base_status == "warn" else "new_warning" + # head failed + if blocking or base_status != "fail": + return "regression" + if set(head_failing) - set(base_failing): + return "regression" + return "pre_existing" + + +def compare_metrics(base: Mapping, head: Mapping, spec: Spec, years) -> tuple: + rows, flags = [], [] + ids = list(dict.fromkeys(list(head) + list(base))) + for metric_id in ids: + head_metric, base_metric = head.get(metric_id), base.get(metric_id) + template = head_metric or base_metric + row = { + "id": metric_id, + "label": template["label"], + "section": template["section"], + "group": template.get("group"), + "kind": template["kind"], + "unit": template.get("unit"), + "values": {}, + # A figure measured on one side only is a change, never "no change". + "changed": not (head_metric and base_metric), + "only_in": None + if head_metric and base_metric + else ("head" if head_metric else "base"), + } + kind = FLAG_KIND_BY_SECTION.get(template["section"], template["kind"]) + threshold = spec.threshold(kind) + for year in years: + key = str(year) + b = base_metric["values"].get(key) if base_metric else None + h = head_metric["values"].get(key) if head_metric else None + delta = h - b if _finite(b) and _finite(h) else None + row["values"][key] = {"base": b, "head": h, "delta": delta} + if delta is not None and delta != 0: + row["changed"] = True + if delta is not None and threshold is not None and threshold.exceeded(b, h): + flags.append( + { + "id": metric_id, + "label": template["label"], + "section": template["section"], + "kind": template["kind"], + "year": year, + "base": b, + "head": h, + "delta": delta, + "limit": threshold.limit(b), + } + ) + rows.append(row) + return rows, flags + + +def _match(base_ids: np.ndarray, head_ids: np.ndarray) -> Optional[np.ndarray]: + """Indices into head arrays that line up with base households, or None. + + The two sides must hold exactly the same households, each once. + """ + base_ids, head_ids = np.asarray(base_ids), np.asarray(head_ids) + if base_ids.shape != head_ids.shape: + return None + if np.array_equal(base_ids, head_ids): + if len(np.unique(base_ids)) != len(base_ids): + return None + return np.arange(len(head_ids)) + if len(np.unique(base_ids)) != len(base_ids) or len(np.unique(head_ids)) != len( + head_ids + ): + return None + order = np.argsort(head_ids, kind="stable") + positions = np.clip(np.searchsorted(head_ids[order], base_ids), 0, len(order) - 1) + matched = order[positions] + if not np.array_equal(head_ids[matched], base_ids): + return None + return matched + + +def decile_impacts(base_households: Mapping, head_households: Mapping, years) -> tuple: + """Household-matched income changes within the base run's deciles. + + For each year and decile d of the base run: the mean change in the income + variable per household and the change relative to the decile's base + income, both weighted by the base household weights; plus the shares of + people (household weight times household size) whose household gains or + loses more than £1 a year. + """ + impacts, notes = {}, [] + for year in years: + key = str(year) + b, h = base_households.get(key), head_households.get(key) + if b is None or h is None: + notes.append(f"{fiscal_year_label(year)}: household detail missing") + continue + matched = _match(np.asarray(b["household_id"]), np.asarray(h["household_id"])) + if matched is None: + notes.append(f"{fiscal_year_label(year)}: base and head households differ") + continue + weight = np.asarray(b["household_weight"], dtype=np.float64) + head_weight = np.asarray(h["household_weight"], dtype=np.float64)[matched] + if not np.array_equal(weight, head_weight): + notes.append( + f"{fiscal_year_label(year)}: household weights changed; decile " + "changes use the base weights" + ) + people = np.asarray(b["household_count_people"], dtype=np.float64) + decile = np.asarray(b["decile"]) + base_income = np.asarray(b["income"], dtype=np.float64) + change = np.asarray(h["income"], dtype=np.float64)[matched] - base_income + rows = [] + for d in DECILES + ("all",): + mask = np.ones_like(decile, dtype=bool) if d == "all" else decile == d + w = weight[mask] + total_weight = float(w.sum()) + base_total = float(np.sum(w * base_income[mask])) + change_total = float(np.sum(w * change[mask])) + person_weight = w * people[mask] + person_total = float(person_weight.sum()) + rows.append( + { + "decile": d, + "households": total_weight, + "base_mean": base_total / total_weight + if total_weight + else float("nan"), + "mean_change": change_total / total_weight + if total_weight + else float("nan"), + "relative_change": change_total / base_total + if base_total + else float("nan"), + "share_gaining": float(np.sum(person_weight * (change[mask] > 1))) + / person_total + if person_total + else float("nan"), + "share_losing": float(np.sum(person_weight * (change[mask] < -1))) + / person_total + if person_total + else float("nan"), + } + ) + excluded = float(np.sum(weight[~np.isin(decile, DECILES)])) + if excluded: + notes.append( + f"{fiscal_year_label(year)}: {excluded:,.0f} households with negative " + "income have no decile and appear only in 'all'" + ) + impacts[key] = rows + return impacts, notes + + +def _runtime_differences(base: Mapping, head: Mapping) -> list: + def listing(result): + return { + line.split("==")[0]: line + for line in (result.get("runtime") or {}).get("distributions", []) + } + + b, h = listing(base), listing(head) + differences = [] + for name in sorted(set(b) | set(h)): + if name == "policyengine-uk": + continue + if b.get(name) != h.get(name): + differences.append( + f"{b.get(name, f'{name} absent')} -> {h.get(name, f'{name} absent')}" + ) + for field in ("python", "machine"): + bv, hv = ( + (base.get("runtime") or {}).get(field), + (head.get("runtime") or {}).get(field), + ) + if bv != hv: + differences.append(f"{field}: {bv} -> {hv}") + return differences + + +def compare( + base: Optional[Mapping], + head: Optional[Mapping], + spec: Spec, + base_households: Optional[Mapping] = None, + head_households: Optional[Mapping] = None, + strict: bool = False, + require_base: bool = True, +) -> dict: + """Compare two measurements. + + With ``require_base`` (the default) a missing or crashed base fails the + comparison: without it there is nothing to show the change against. + """ + failures, warnings = [], [] + if head and head.get("status") == "skipped": + return { + "schema": TOOL_SCHEMA + "/comparison", + "status": "skipped", + "failures": [], + "warnings": [f"Not measured: {head.get('reason', 'no reason recorded')}"], + "years": spec.years, + "checks": [], + "rows": [], + "flags": [], + "deciles": {}, + "decile_notes": [], + "decile_flags": [], + "runtime_differences": [], + "base": _side_summary(base), + "head": _side_summary(head), + } + head_ok = bool(head) and head.get("status") == "ok" + base_ok = bool(base) and base.get("status") == "ok" + if not head: + failures.append( + "The head measurement produced no result; see the measure job's log." + ) + elif not head_ok: + failures.append( + "The head measurement crashed: " + _last_line(head.get("error")) + ) + base_problem = None + if not base: + base_problem = "The base measurement produced no result" + elif not base_ok: + base_problem = "The base measurement crashed: " + _last_line(base.get("error")) + if base_problem and require_base: + failures.append(base_problem + "; the change cannot be compared without it.") + elif base_problem: + warnings.append(base_problem + "; the head's own checks decide.") + + years = (head or base or {}).get("years") or spec.years + if head_ok and base_ok: + if base.get("years") != head.get("years"): + failures.append( + f"Base and head measured different years ({base.get('years')} vs {head.get('years')})." + ) + b_sha = (base.get("provenance") or {}).get("dataset", {}).get("verified_sha256") + h_sha = (head.get("provenance") or {}).get("dataset", {}).get("verified_sha256") + if not b_sha or not h_sha or b_sha != h_sha: + failures.append( + f"Base and head did not run on the same verified dataset ({b_sha} vs {h_sha})." + ) + if (base.get("partition") or {}).get("sha256") != ( + head.get("partition") or {} + ).get("sha256"): + warnings.append( + "This change edits the fiscal partition; review the partition diff alongside the moves it explains." + ) + runtime_differences = _runtime_differences(base, head) + if runtime_differences: + warnings.append( + "Base and head ran on different runtimes, so some changes may come from dependencies: " + + "; ".join(runtime_differences[:10]) + ) + else: + runtime_differences = [] + + base_checks, head_checks = _check_map(base), _check_map(head) + same_partition = ( + base_ok + and head_ok + and (base.get("partition") or {}).get("sha256") + == (head.get("partition") or {}).get("sha256") + ) + checks = [] + if head_ok: + missing = [c for c in REQUIRED_CHECKS if c not in head_checks] + if missing: + failures.append( + "The head measurement did not run: " + ", ".join(missing) + "." + ) + for check_id in dict.fromkeys(list(head_checks) + list(base_checks)): + b, h = base_checks.get(check_id), head_checks.get(check_id) + blocking = check_id.startswith(ALWAYS_BLOCKING_PREFIXES) or not base_ok + classification = classify_check( + b["status"] if b else None, + h["status"] if h else None, + blocking, + (b or {}).get("failing", {}), + (h or {}).get("failing", {}), + ) + if strict and classification == "pre_existing": + classification = "regression" + if classification == "skipped" and b and b["status"] != "skipped": + warnings.append( + f"The check {check_id} no longer applies on the head: " + + "; ".join((h or {}).get("details", [])[:1]) + ) + if classification == "removed" and head_ok: + # A check can disappear with the head it checked; otherwise its + # disappearance would let a failure pass unseen. + if same_partition: + failures.append( + f"The check {check_id} ran on the base but not on the head." + ) + else: + warnings.append( + f"The check {check_id} ran on the base but not on the head " + "(the partition changed)." + ) + template = h or b + checks.append( + { + "id": check_id, + "label": template["label"], + "section": template.get("section", ""), + "base_status": b["status"] if b else None, + "head_status": h["status"] if h else None, + "classification": classification, + "head_details": (h or {}).get("details", []), + "base_details": (b or {}).get("details", []), + } + ) + regressions = [c for c in checks if c["classification"] == "regression"] + for check in regressions: + failures.append( + f"{check['label']} ({check['id']}): " + "; ".join(check["head_details"][:3]) + ) + + rows, flags = compare_metrics(_metric_map(base), _metric_map(head), spec, years) + non_finite = [] + for row in rows: + if row["only_in"] == "base" and head_ok: + warnings.append( + f"{row['label']} ({row['id']}) is measured on the base only." + ) + elif row["only_in"] == "head" and base_ok: + warnings.append( + f"{row['label']} ({row['id']}) is measured on the head only." + ) + if row["only_in"] != "base": + non_finite += [ + f"{row['id']} {fiscal_year_label(int(year))}" + for year, cell in row["values"].items() + if not _finite(cell["head"]) + ] + if non_finite and head_ok: + more = f" and {len(non_finite) - 10} more" if len(non_finite) > 10 else "" + failures.append( + "The head has figures that are missing or not finite: " + + "; ".join(non_finite[:10]) + + more + + "." + ) + + deciles, decile_notes, decile_flags = {}, [], [] + if head_ok and base_ok and base_households and head_households: + deciles, decile_notes = decile_impacts(base_households, head_households, years) + threshold = spec.threshold("decile_relative") + for year, decile_rows in deciles.items(): + for row in decile_rows: + rel = row["relative_change"] + if ( + threshold is not None + and _finite(rel) + and abs(rel) > threshold.limit(0) + ): + decile_flags.append({"year": int(year), **row}) + + return { + "schema": TOOL_SCHEMA + "/comparison", + "status": "fail" if failures else "pass", + "failures": failures, + "warnings": warnings, + "years": years, + "checks": checks, + "rows": rows, + "flags": flags, + "deciles": deciles, + "decile_notes": decile_notes, + "decile_flags": decile_flags, + "runtime_differences": runtime_differences, + "base": _side_summary(base), + "head": _side_summary(head), + } + + +def _last_line(text) -> str: + lines = [line for line in str(text or "").strip().splitlines() if line.strip()] + return lines[-1] if lines else "no error recorded" + + +def _side_summary(result: Optional[Mapping]) -> Optional[dict]: + if not result: + return None + return { + "status": result.get("status"), + "label": result.get("label"), + "provenance": result.get("provenance"), + "runtime": { + k: v + for k, v in (result.get("runtime") or {}).items() + if k != "distributions" + }, + "partition": { + k: (result.get("partition") or {}).get(k) for k in ("source", "sha256") + }, + "guard": result.get("guard"), + "reforms": result.get("reforms"), + "timings": result.get("timings"), + "error": result.get("error"), + } diff --git a/policyengine_uk/utils/aggregate_impact/dataset.py b/policyengine_uk/utils/aggregate_impact/dataset.py new file mode 100644 index 0000000000..bdd32173bf --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/dataset.py @@ -0,0 +1,164 @@ +"""Pinned dataset selection: load a pin, fetch its file, verify its bytes.""" + +from __future__ import annotations + +import hashlib +import os +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Optional, Union + +import yaml + +_HEX40 = re.compile(r"^[0-9a-f]{40}$") +_HEX64 = re.compile(r"^[0-9a-f]{64}$") +_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$") + + +class DatasetPinError(ValueError): + """The pin file is malformed or names a mutable revision.""" + + +class DatasetMismatchError(RuntimeError): + """A dataset file's bytes do not match the pinned SHA-256.""" + + +@dataclass(frozen=True) +class DatasetPin: + """An immutable reference to one dataset file on the Hugging Face Hub.""" + + repo_id: str + path: str + revision: str + sha256: str + data_package: str = "" + data_version: str = "" + hf_tag: Optional[str] = None + size_bytes: Optional[int] = None + certification: Mapping[str, str] = field(default_factory=dict) + + @property + def uri(self) -> str: + """The ``hf://`` reference ``policyengine_uk.Simulation`` accepts.""" + return f"hf://{self.repo_id}/{self.path}@{self.revision}" + + def problems(self) -> list[str]: + """Every reason this pin is not an immutable, verifiable reference.""" + problems = [] + if not _REPO_ID.match(self.repo_id or ""): + problems.append(f"repo_id {self.repo_id!r} is not 'owner/name'") + if not self.path or "@" in self.path or self.path.startswith("/"): + problems.append(f"path {self.path!r} is not a relative file path") + if not _HEX40.match(self.revision or ""): + problems.append( + f"revision {self.revision!r} is not a 40-character commit hash; " + "tags and branches can move" + ) + if not _HEX64.match(self.sha256 or ""): + problems.append(f"sha256 {self.sha256!r} is not a lowercase hex digest") + if self.size_bytes is not None and self.size_bytes <= 0: + problems.append(f"size_bytes {self.size_bytes!r} is not positive") + return problems + + def describe(self) -> str: + version = f"{self.data_package} {self.data_version}".strip() + return ( + f"{self.path} ({version}, commit {self.revision[:8]}, " + f"sha256 {self.sha256[:12]})" + ) + + def as_dict(self) -> dict: + return { + "uri": self.uri, + "repo_id": self.repo_id, + "path": self.path, + "revision": self.revision, + "sha256": self.sha256, + "data_package": self.data_package, + "data_version": self.data_version, + "hf_tag": self.hf_tag, + "size_bytes": self.size_bytes, + "certification": dict(self.certification), + } + + +def load_pin(path: Union[str, os.PathLike]) -> DatasetPin: + """Read a pin file and reject it unless it is immutable and complete.""" + with open(path) as f: + raw = yaml.safe_load(f) or {} + known = { + "repo_id", + "path", + "revision", + "sha256", + "data_package", + "data_version", + "hf_tag", + "size_bytes", + "certification", + } + unknown = sorted(set(raw) - known) + if unknown: + raise DatasetPinError(f"{path}: unknown keys {unknown}") + pin = DatasetPin( + repo_id=str(raw.get("repo_id", "")), + path=str(raw.get("path", "")), + revision=str(raw.get("revision", "")), + sha256=str(raw.get("sha256", "")), + data_package=str(raw.get("data_package", "")), + data_version=str(raw.get("data_version", "")), + hf_tag=None if raw.get("hf_tag") is None else str(raw["hf_tag"]), + size_bytes=None if raw.get("size_bytes") is None else int(raw["size_bytes"]), + certification={ + str(k): str(v) for k, v in (raw.get("certification") or {}).items() + }, + ) + problems = pin.problems() + if problems: + raise DatasetPinError(f"{path}: " + "; ".join(problems)) + return pin + + +def sha256_file(path: Union[str, os.PathLike], chunk_size: int = 1 << 20) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def verify_file(path: Union[str, os.PathLike], expected_sha256: str) -> str: + """Return the file's SHA-256, raising if it differs from the expected one.""" + actual = sha256_file(path) + if actual != expected_sha256: + raise DatasetMismatchError( + f"{path} has sha256 {actual}, but the pin requires {expected_sha256}" + ) + return actual + + +def materialise( + pin: DatasetPin, + token: Optional[str] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, +) -> Path: + """Download the pinned file (or reuse the cached copy) and verify it. + + The token defaults to ``HUGGING_FACE_TOKEN``. The Hugging Face cache is + keyed by commit, so a verified file is never downloaded twice. + """ + from huggingface_hub import hf_hub_download + + if token is None: + token = os.environ.get("HUGGING_FACE_TOKEN") or None + local = hf_hub_download( + repo_id=pin.repo_id, + filename=pin.path, + revision=pin.revision, + repo_type="model", + token=token, + cache_dir=cache_dir, + ) + verify_file(local, pin.sha256) + return Path(local) diff --git a/policyengine_uk/utils/aggregate_impact/guard.py b/policyengine_uk/utils/aggregate_impact/guard.py new file mode 100644 index 0000000000..c209c3d53f --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/guard.py @@ -0,0 +1,336 @@ +"""The counted-once check: every component enters each total exactly once. + +For each component of a partition total, the check adds 1 to every unit +where the component is non-zero, recomputes every head of the totals it +belongs to, and reads each head's response household by household. For a +total T and a component c: + +- c's own head (c itself, or the head a declared double count is contained + in) must rise by exactly 1 per unit added; +- on a tax total, no other head may move at all: a head that moves contains + c, or reads it; +- on a spending total, no other head may rise in any household. Means tests + may lower other benefits when one rises (Pension Credit falls as State + Pension rises), but a benefit that rises with another either contains it + or pays more to people with more income. Rises and falls are not netted, + so a fall elsewhere cannot hide a double count; +- the model's own total must move by exactly what its heads moved, plus one + more per unit if c is a declared double count of T. This catches a total + whose formula adds a component twice even when that component is zero. + +A head that declares a read of c's head (``reads`` in the partition) may +respond, unless it responds one for one, which is containment. A spending +component that is zero for every unit is perturbed everywhere, which makes +recipients of non-recipients; other heads may then rise through passporting, +so only a one-for-one response fails. + +The check reuses one simulation. It takes its snapshot of supplied inputs +only after discarding formula results, so a value some formula computed +earlier is never frozen as an input. Between perturbations it discards every +formula result, and restores any input it overrode; at the end it recomputes +the baseline and requires it to match the first computation exactly. +""" + +from __future__ import annotations + +from typing import Iterable, Mapping, Optional + +import numpy as np + +from .partition import Partition + + +def _raw(sim, variable, period): + try: + result = sim.calculate(variable, period, unweighted=True) + except TypeError: + result = sim.calculate(variable, period) + return np.asarray(getattr(result, "values", result)) + + +def discard_formula_results(sim) -> bool: + """Drop every calculated value, keeping supplied inputs. False if unsupported.""" + reset = getattr(sim, "_invalidate_all_caches", None) + if reset is None: + return False + reset() + return True + + +def snapshot_inputs(sim, variables) -> dict: + """Copy the supplied input arrays of ``variables``. + + Formula results are discarded first: a holder's known periods include + calculated values, and restoring one of those as an input would freeze + it for every later perturbation. + """ + discard_formula_results(sim) + snapshot = {} + for variable in variables: + if variable not in sim.tax_benefit_system.variables: + continue + holder = sim.get_holder(variable) + periods = holder.get_known_periods() + if periods: + snapshot[variable] = [ + (period, np.array(holder.get_array(period), copy=True)) + for period in periods + ] + return snapshot + + +def _restore(sim, variable, snapshot): + sim.delete_arrays(variable) + for period, values in snapshot.get(variable, []): + sim.set_input(variable, period, values) + + +def _months(year): + from policyengine_core.periods import period as make_period + + return make_period(str(year)).get_subperiods("month") + + +def _household(sim, values, entity): + values = np.asarray(values, dtype=np.float64) + if entity == "household": + return values + return np.asarray(sim.map_result(values, entity, "household"), dtype=np.float64) + + +def _head_arrays(sim, year, variables) -> dict: + """Each variable's values summed to households.""" + definitions = sim.tax_benefit_system.variables + return { + v: _household(sim, _raw(sim, v, year), definitions[v].entity.key) + for v in variables + } + + +def judge_total( + *, + total: str, + is_tax: bool, + component: str, + own_variable: str, + declared_double_count: bool, + zero_baseline: bool, + weights: np.ndarray, + expected: np.ndarray, + head_changes: Mapping[str, np.ndarray], + model_change: Optional[np.ndarray], + allowed_readers: Iterable[str], + tolerance: float, +) -> dict: + """Apply the counted-once rules to one total's responses. + + ``head_changes`` maps each head variable of the total to its change by + household; ``expected`` is what was added to the component by household. + Returns the responses per unit added, the problem with the component's + own head (if it did not rise one for one) and the other problems. + """ + added = float(np.sum(weights * expected)) + allowed = set(allowed_readers) + responses, problems, own_problem = {}, [], None + for variable, change in head_changes.items(): + net = float(np.sum(weights * change)) / added + rise = float(np.sum(weights * np.maximum(change, 0.0))) / added + moved = float(np.sum(weights * np.abs(change))) / added + responses[variable] = net + if variable == own_variable: + off = float(np.sum(weights * np.abs(change - expected))) / added + if off > tolerance: + own_problem = f"{total}: {variable} moves by {net:.4f} per unit of {component}, not 1" + continue + one_for_one = abs(net - 1) <= tolerance and abs(rise - 1) <= tolerance + if variable in allowed or (zero_baseline and not is_tax): + if one_for_one: + problems.append( + f"{total}: {variable} rises one for one with {component}, so it contains it" + ) + elif is_tax: + if moved > tolerance: + problems.append( + f"{total}: {variable} moves by {net:+.4f} per unit of {component}" + ) + elif rise > tolerance: + problems.append( + f"{total}: {variable} rises by {rise:.4f} per unit of {component}" + ) + partition_coefficient = float(sum(responses.values())) + model_coefficient = None + if model_change is not None: + model_coefficient = float(np.sum(weights * model_change)) / added + expected_model = partition_coefficient + (1.0 if declared_double_count else 0.0) + if abs(model_coefficient - expected_model) > tolerance: + problems.append( + f"the model's {total} moves by {model_coefficient:.4f} per unit of " + f"{component}, but its heads account for {expected_model:.4f}" + ) + return { + "responses": responses, + "coefficient": partition_coefficient, + "model_coefficient": model_coefficient, + "own_problem": own_problem, + "problems": problems, + } + + +def counted_once( + sim, + partition: Partition, + year: int, + tolerance: float, + snapshot: Optional[Mapping] = None, + active: Iterable[str] = (), + variables_to_check: Optional[Iterable[str]] = None, +) -> dict: + """Perturb each component (or those in ``variables_to_check``) once.""" + if not discard_formula_results(sim): + return { + "status": "skipped", + "reason": "this policyengine-core has no way to discard formula results", + } + snapshot = snapshot or {} + active = set(active) + definitions = sim.tax_benefit_system.variables + totals = [t for t in partition.totals if partition.heads_in(t, active)] + heads_of = {t: [h.variable for h in partition.heads_in(t, active)] for t in totals} + all_heads = list(dict.fromkeys(v for t in totals for v in heads_of[t])) + model_totals = [t for t in totals if t in definitions] + + hw = _raw(sim, "household_weight", year).astype(np.float64) + heads0 = _head_arrays(sim, year, all_heads) + model0 = {t: _raw(sim, t, year).astype(np.float64) for t in model_totals} + baseline = {} + for variable in partition.component_variables(): + if definitions[variable].definition_period == "month": + baseline[variable] = [ + (month, _raw(sim, variable, month).astype(np.float64)) + for month in _months(year) + ] + else: + baseline[variable] = _raw(sim, variable, year).astype(np.float64) + + declared = {dc.variable: dc for dc in partition.double_counts} + head_by_variable = {h.variable: h for h in partition.heads} + selected = None if variables_to_check is None else set(variables_to_check) + results, double_count_results = [], [] + for variable in partition.component_variables(): + if selected is not None and variable not in selected: + continue + definition = definitions[variable] + entity = definition.entity.key + in_totals = [t for t in totals if variable in partition.components(t, active)] + record = { + "variable": variable, + "totals": in_totals, + "coefficients": {}, + "model_coefficients": {}, + "responses": {}, + "units": 0, + "status": "pass", + "detail": "", + } + if definition.value_type is not float: + record.update(status="skipped", detail="not a money variable") + results.append(record) + continue + monthly = definition.definition_period == "month" + year_values = ( + sum(values for _, values in baseline[variable]) + if monthly + else baseline[variable] + ) + nonzero = year_values != 0 + zero_baseline = not bool(nonzero.any()) + mask = np.ones_like(nonzero) if zero_baseline else nonzero + added = mask.astype(np.float64) + expected = _household(sim, added, entity) + record["units"] = int(mask.sum()) + own = declared[variable].contained_in if variable in declared else variable + own_head = head_by_variable[own] + + discard_formula_results(sim) + if monthly: + for month, values in baseline[variable]: + sim.set_input(variable, month, values + added / 12) + else: + sim.set_input(variable, year, baseline[variable] + added) + needed = list(dict.fromkeys(v for t in in_totals for v in heads_of[t])) + heads1 = _head_arrays(sim, year, needed) + problems = [] + for total in in_totals: + readers = [ + v for v in heads_of[total] if own_head.id in head_by_variable[v].reads + ] + verdict = judge_total( + total=total, + is_tax=partition.is_tax_total(total), + component=variable, + own_variable=own, + declared_double_count=variable in declared + and total in declared[variable].totals, + zero_baseline=zero_baseline, + weights=hw, + expected=expected, + head_changes={v: heads1[v] - heads0[v] for v in heads_of[total]}, + model_change=( + _raw(sim, total, year).astype(np.float64) - model0[total] + if total in model0 + else None + ), + allowed_readers=readers, + tolerance=tolerance, + ) + record["coefficients"][total] = verdict["coefficient"] + if verdict["model_coefficient"] is not None: + record["model_coefficients"][total] = verdict["model_coefficient"] + record["responses"][total] = { + v: r for v, r in verdict["responses"].items() if abs(r) > tolerance + } + problems += verdict["problems"] + # A declared double count whose container no longer moves is + # reported once, as a stale declaration, below. + if verdict["own_problem"] and variable not in declared: + problems.append(verdict["own_problem"]) + if variable in declared: + dc = declared[variable] + container_change = heads1[own] - heads0[own] + off = float(np.sum(hw * np.abs(container_change - expected))) / float( + np.sum(hw * expected) + ) + k = float(np.sum(hw * container_change)) / float(np.sum(hw * expected)) + present = off <= tolerance + double_count_results.append( + { + "variable": variable, + "contained_in": dc.contained_in, + "coefficient": k, + "issue": dc.issue, + "status": "pass" if present else "fail", + "detail": "" + if present + else ( + f"{dc.contained_in} moves by {k:.4f} per unit of {variable}: " + "the model no longer counts it twice, so remove the " + "declaration and give it its own head" + ), + } + ) + if problems: + record.update(status="fail", detail="; ".join(problems)) + results.append(record) + _restore(sim, variable, snapshot) + + discard_formula_results(sim) + heads_again = _head_arrays(sim, year, all_heads) + reproduces = all(np.array_equal(heads_again[v], heads0[v]) for v in all_heads) + return { + "status": "ran", + "year": year, + "tolerance": tolerance, + "components": results, + "double_counts": double_count_results, + "reset_reproduces_baseline": bool(reproduces), + } diff --git a/policyengine_uk/utils/aggregate_impact/measure.py b/policyengine_uk/utils/aggregate_impact/measure.py new file mode 100644 index 0000000000..7b0dff4d98 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/measure.py @@ -0,0 +1,941 @@ +"""Measure one model checkout on one dataset over the report horizon.""" + +from __future__ import annotations + +import gc +import hashlib +import json +import platform +import re +import sys +import time +from importlib import metadata +from pathlib import Path +from typing import Callable, Mapping, Optional, Sequence + +import numpy as np + +from . import TOOL_SCHEMA +from .guard import counted_once, snapshot_inputs +from .partition import Partition, closure_problems, group_label, read_model_lists +from .reforms import StandardReform, parse_reforms, reform_parameters, responsiveness +from .spec import Spec, fiscal_year_label + +ENTITIES = ("person", "benunit", "household") + +# Model totals reported alongside the partition. Each is a household variable. +HOUSEHOLD_TOTALS = { + "household_market_income": "Household market income", + "household_net_income": "Household net income", + "hbai_household_net_income": "Household net income (HBAI definition)", +} + + +class Recorder: + """Collects metric values and check outcomes for one measurement.""" + + def __init__(self): + self.metrics: dict = {} + self.checks: dict = {} + # The year being measured; a check records which scopes it failed + # in, so a comparison can tell a newly failing year from an old one. + self.scope: str = "" + + def metric( + self, metric_id, label, section, kind, year, value, group=None, unit=None + ): + record = self.metrics.setdefault( + metric_id, + { + "id": metric_id, + "label": label, + "section": section, + "group": group, + "kind": kind, + "unit": unit, + "values": {}, + }, + ) + record["values"][str(year)] = float(value) + + def check( + self, check_id, label, ok, detail="", severity="fail", section="", scope=None + ): + """Record a check; repeated ids (one per year) merge into one result. + + ``failing`` maps each scope the check failed in (a year, or "" for a + check that is not per year) to its detail. ``ok=None`` records that + the check did not apply (``skipped``), which is not a pass. + """ + if ok is None: + status = "skipped" + else: + status = "pass" if ok else ("warn" if severity == "warn" else "fail") + scope = self.scope if scope is None else scope + record = self.checks.get(check_id) + if record is None: + record = { + "id": check_id, + "label": label, + "section": section, + "severity": severity, + "status": status, + "details": [], + "failing": {}, + } + self.checks[check_id] = record + elif status in ("fail", "warn") and record["status"] in ("pass", "skipped"): + record["status"] = status + elif status == "pass" and record["status"] == "skipped": + record["status"] = status + if ok is None: + if detail and detail not in record["details"]: + record["details"].append(detail) + elif not ok: + record["failing"][scope] = detail + if detail: + record["details"].append(detail) + + +def array(sim, variable: str, period, map_to: Optional[str] = None) -> np.ndarray: + """A variable's raw values, without survey weights.""" + try: + result = sim.calculate(variable, period, map_to=map_to, unweighted=True) + except TypeError: + result = sim.calculate(variable, period, map_to=map_to) + return np.asarray(getattr(result, "values", result)) + + +def active_parameters(sim, partition: Partition, year) -> set: + """The ``dropped_when`` switches that are on in ``year``.""" + active = set() + for path in partition.dropped_when_parameters(): + node = sim.tax_benefit_system.parameters + try: + for part in path.split("."): + node = getattr(node, part) + if bool(node(f"{year}-01-01")): + active.add(path) + except AttributeError: + continue + return active + + +def entity_of(sim, variable: str) -> str: + return sim.tax_benefit_system.variables[variable].entity.key + + +def to_household(sim, values: np.ndarray, entity: str) -> np.ndarray: + if entity == "household": + return np.asarray(values, dtype=np.float64) + return np.asarray( + sim.map_result(np.asarray(values, dtype=np.float64), entity, "household"), + dtype=np.float64, + ) + + +def weighted_gini(values: np.ndarray, weights: np.ndarray) -> float: + """Gini coefficient of ``values`` with non-negative ``weights``.""" + values = np.asarray(values, dtype=np.float64) + weights = np.asarray(weights, dtype=np.float64) + keep = weights > 0 + values, weights = values[keep], weights[keep] + if values.size == 0: + return float("nan") + order = np.argsort(values, kind="stable") + values, weights = values[order], weights[order] + cumulative_weight = np.cumsum(weights) + cumulative_income = np.cumsum(weights * values) + total_weight, total_income = cumulative_weight[-1], cumulative_income[-1] + if total_income == 0: + return float("nan") + previous = np.concatenate([[0.0], cumulative_income[:-1]]) + area = np.sum(weights * (cumulative_income + previous)) / ( + total_weight * total_income + ) + return float(1.0 - area) + + +def slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def _digest_tool_files() -> str: + root = Path(__file__).resolve().parent + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if path.suffix in (".py", ".yaml") and "__pycache__" not in path.parts: + digest.update(path.relative_to(root).as_posix().encode() + b"\0") + digest.update(path.read_bytes() + b"\0") + return digest.hexdigest() + + +# Taken when this module loads, so a result is stamped with the code that +# produced it even if the files change while a measurement runs. +_TOOL_DIGEST = _digest_tool_files() + + +def tool_digest() -> str: + """SHA-256 of this package's code and spec, independent of where it lives.""" + return _TOOL_DIGEST + + +def definitions_digest(spec: Spec, reforms: Sequence) -> str: + """SHA-256 of what was actually measured: the effective spec and reforms. + + ``tool_digest`` covers the files as shipped; this covers a spec or reform + set passed in instead, and a horizon overridden at the call. + """ + payload = { + "spec": spec.raw, + "years": spec.years, + "guard_year": spec.guard_year, + "reforms": [ + { + "id": r.id, + "parameters": dict(r.parameters), + "scale": dict(r.scale), + "responds": None + if r.responds is None + else [r.responds.variable, r.responds.direction, r.responds.minimum], + } + for r in reforms + ], + } + text = json.dumps(payload, sort_keys=True, default=str) + return hashlib.sha256(text.encode()).hexdigest() + + +def runtime_identity() -> dict: + """The installed distributions and interpreter that ran the measurement.""" + distributions = sorted( + { + (d.metadata["Name"].lower().replace("_", "-"), d.version) + for d in metadata.distributions() + if d.metadata["Name"] + } + ) + listing = "\n".join(f"{name}=={version}" for name, version in distributions) + + def version(name): + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + return { + "python": sys.version.split()[0], + "platform": platform.platform(), + "machine": platform.machine(), + "policyengine_uk": version("policyengine-uk"), + "policyengine_core": version("policyengine-core"), + "numpy": version("numpy"), + "distributions": [f"{n}=={v}" for n, v in distributions], + "distributions_sha256": hashlib.sha256(listing.encode()).hexdigest(), + } + + +def _identity_check(rec, check_id, label, components_hh, total_hh, year, spec, weights): + """Household-level accounting identity: components add up to the total. + + The detail gives counts and weighted totals only: results may be public, + so they never quote one household's value. + """ + difference = components_hh - total_hh + scale = np.abs(total_hh) + 1.0 + tolerance = 1.0 + spec.identity_relative_tolerance * scale + bad = np.abs(difference) > tolerance + finite = np.all(np.isfinite(difference)) + ok = bool(finite and not bad.any()) + detail = "" + if not ok: + gap = float(np.nansum(weights * np.where(bad, difference, 0.0))) + detail = ( + f"{fiscal_year_label(year)}: {int(bad.sum())} households differ, " + f"by a weighted £{gap:,.0f} in total" + ) + rec.check(check_id, label, ok, detail, section="identities") + + +def measure_year( + sim, partition: Partition, spec: Spec, year: int, rec: Recorder, households: dict +): + tbs_variables = sim.tax_benefit_system.variables + weights = {e: array(sim, f"{e}_weight", year).astype(np.float64) for e in ENTITIES} + for entity, label in ( + ("person", "People"), + ("benunit", "Benefit units"), + ("household", "Households"), + ): + w = weights[entity] + rec.check( + f"weights:{entity}", + f"{label} weights are finite and non-negative", + bool(np.all(np.isfinite(w)) and np.all(w >= 0)), + f"{fiscal_year_label(year)}: invalid {entity} weights", + section="data", + ) + rec.metric(f"population.{entity}", label, "population", "count", year, w.sum()) + hw = weights["household"] + + head_hh = {} + for head in partition.heads: + entity = entity_of(sim, head.variable) + values = array(sim, head.variable, year).astype(np.float64) + finite = np.isfinite(values) + rec.check( + f"finite:{head.id}", + f"{head.label} is finite for every unit", + bool(finite.all()), + f"{fiscal_year_label(year)}: {int((~finite).sum())} non-finite values", + section="finite", + ) + hh = to_household(sim, values, entity) + head_hh[head.variable] = hh + section = "receipts" if head.kind == "tax" else "spending" + rec.metric( + f"{section}.{head.id}", + head.label, + section, + "gbp", + year, + float(np.sum(hw * hh)), + group=group_label(head.report_group), + ) + if head.kind == "spending" and head.caseload: + rec.metric( + f"caseloads.{head.id}", + head.label, + "caseloads", + "count", + year, + float(np.sum(weights[entity] * (values > 0))), + group=group_label(head.report_group), + unit=entity, + ) + if head.sign == "nonnegative": + negative = values < -0.005 + rec.check( + f"sign:{head.id}", + f"{head.label} is never negative", + not bool(negative.any()), + f"{fiscal_year_label(year)}: {int(negative.sum())} units below zero, " + f"a weighted £{float(np.sum(hw * np.where(hh < 0, hh, 0.0))):,.0f} in total", + section="signs", + ) + if not head.zero_at_baseline: + rec.check( + f"inert:{head.id}", + f"{head.label} is not zero for every unit", + bool(np.any(values != 0)), + f"{fiscal_year_label(year)}: zero for every unit", + severity="warn", + section="coverage", + ) + + variable_of = {head.id: head.variable for head in partition.heads} + for guard in partition.overlap_guards: + if ( + not isinstance(guard, Mapping) + or guard.get("kind") != "geographic_exclusivity" + ): + continue + members = [variable_of[h] for h in guard.get("heads") or [] if h in variable_of] + nonzero = sum((head_hh[v] != 0).astype(int) for v in members) + overlapping = nonzero > 1 + rec.check( + f"overlap_guard:{guard.get('id')}", + "At most one of " + ", ".join(members) + " is non-zero for any household", + not bool(overlapping.any()), + f"{fiscal_year_label(year)}: {int(overlapping.sum())} households have more " + f"than one, a weighted {float(np.sum(hw * overlapping)):,.0f}", + section="partition", + ) + + double_count_hh = {} + for dc in partition.double_counts: + values = array(sim, dc.variable, year).astype(np.float64) + hh = to_household(sim, values, entity_of(sim, dc.variable)) + double_count_hh[dc.variable] = hh + rec.metric( + f"double_counts.{dc.variable}", + f"{dc.variable}, counted again inside {dc.contained_in}", + "double_counts", + "gbp", + year, + float(np.sum(hw * hh)), + ) + + active = active_parameters(sim, partition, year) + model_hh = {} + for total in partition.totals: + if total not in tbs_variables: + continue + hh = to_household( + sim, array(sim, total, year).astype(np.float64), entity_of(sim, total) + ) + model_hh[total] = hh + rec.metric( + f"model.{total}", total, "model_totals", "gbp", year, float(np.sum(hw * hh)) + ) + components = np.zeros_like(hh) + for head in partition.heads_in(total, active): + components = components + head_hh[head.variable] + for dc in partition.double_counts_in(total, active): + components = components + double_count_hh[dc.variable] + _identity_check( + rec, + f"identity:{total}", + f"The partition's heads and declared double counts add up to {total}", + components, + hh, + year, + spec, + hw, + ) + + def partition_sum(total): + out = np.zeros_like(hw) + for head in partition.heads_in(total, active): + out = out + head_hh[head.variable] + return out + + receipts = partition_sum("gov_tax") + spending = partition_sum("gov_spending") + rec.metric( + "balance.receipts", + "Government receipts, each component once", + "balance", + "gbp", + year, + float(np.sum(hw * receipts)), + ) + rec.metric( + "balance.spending", + "Government spending", + "balance", + "gbp", + year, + float(np.sum(hw * spending)), + ) + rec.metric( + "balance.balance", + "Government balance, each receipt once", + "balance", + "gbp", + year, + float(np.sum(hw * (receipts - spending))), + ) + household_only = np.zeros_like(hw) + for head in partition.spending_heads: + if "household_benefits" in head.totals and "gov_spending" not in head.totals: + household_only = household_only + head_hh[head.variable] + rec.metric( + "balance.household_only_benefits", + "Benefits in household_benefits but not in gov_spending", + "balance", + "gbp", + year, + float(np.sum(hw * household_only)), + ) + + if {"gov_balance", "gov_tax", "gov_spending"} <= set(tbs_variables): + gov_balance = array(sim, "gov_balance", year).astype(np.float64) + rec.metric( + "model.gov_balance", + "gov_balance", + "model_totals", + "gbp", + year, + float(np.sum(hw * gov_balance)), + ) + _identity_check( + rec, + "identity:gov_balance", + "gov_balance equals gov_tax minus gov_spending", + model_hh.get("gov_tax", array(sim, "gov_tax", year)) + - model_hh.get("gov_spending", array(sim, "gov_spending", year)), + gov_balance, + year, + spec, + hw, + ) + for variable, label in HOUSEHOLD_TOTALS.items(): + if variable in tbs_variables: + values = array(sim, variable, year).astype(np.float64) + rec.metric( + f"household.{variable}", + label, + "household_totals", + "gbp", + year, + float(np.sum(hw * values)), + ) + if { + "household_net_income", + "household_market_income", + "household_benefits", + "household_tax", + "pension_contributions", + } <= set(tbs_variables): + pension = to_household( + sim, + array(sim, "pension_contributions", year).astype(np.float64), + entity_of(sim, "pension_contributions"), + ) + components = ( + array(sim, "household_market_income", year).astype(np.float64) + + model_hh.get("household_benefits", array(sim, "household_benefits", year)) + - model_hh.get("household_tax", array(sim, "household_tax", year)) + - pension + ) + _identity_check( + rec, + "identity:household_net_income", + "Household net income equals market income plus benefits less taxes and pension contributions", + components, + array(sim, "household_net_income", year).astype(np.float64), + year, + spec, + hw, + ) + + pw = weights["person"] + for measure_spec in spec.poverty_measures: + try: + in_poverty = array( + sim, measure_spec["variable"], year, map_to="person" + ).astype(bool) + except Exception as error: # noqa: BLE001 - recorded, not raised + rec.check( + f"available:{measure_spec['id']}", + f"{measure_spec['label']} can be computed", + False, + f"{fiscal_year_label(year)}: {type(error).__name__}: {error}", + section="coverage", + ) + continue + rec.check( + f"available:{measure_spec['id']}", + f"{measure_spec['label']} can be computed", + True, + section="coverage", + ) + for group in spec.poverty_groups: + if group.get("variable"): + try: + members = array(sim, group["variable"], year).astype(bool) + except Exception as error: # noqa: BLE001 + rec.check( + f"available:{group['id']}", + f"{group['label']} can be identified", + False, + f"{fiscal_year_label(year)}: {type(error).__name__}: {error}", + section="coverage", + ) + continue + rec.check( + f"available:{group['id']}", + f"{group['label']} can be identified", + True, + section="coverage", + ) + else: + members = np.ones_like(in_poverty, dtype=bool) + population = float(np.sum(pw * members)) + poor = float(np.sum(pw * (members & in_poverty))) + rate = poor / population if population > 0 else float("nan") + rec.check( + "bounds:poverty_rates", + "Poverty rates lie between 0 and 1", + bool(0.0 <= rate <= 1.0), + f"{fiscal_year_label(year)}: {measure_spec['id']} {group['id']} is {rate}", + section="bounds", + ) + metric_id = f"{measure_spec['id']}.{group['id']}" + label = f"{measure_spec['label']}: {group['label'].lower()}" + rec.metric( + f"poverty.{metric_id}", + label, + "poverty", + "rate", + year, + rate, + group=measure_spec["label"], + ) + rec.metric( + f"poverty_count.{metric_id}", + label, + "poverty_count", + "count", + year, + poor, + group=measure_spec["label"], + ) + + people = array(sim, "household_count_people", year).astype(np.float64) + for inequality in spec.inequality: + income = array(sim, inequality["variable"], year).astype(np.float64) + rec.metric( + f"inequality.{inequality['id']}", + inequality["label"], + "inequality", + "index", + year, + weighted_gini(income, hw * people), + ) + + decile = array(sim, spec.decile_variable, year).astype(np.int64) + income = array(sim, spec.income_variable, year).astype(np.float64) + for d in range(1, 11): + in_decile = decile == d + weight = float(np.sum(hw[in_decile])) + rec.metric( + f"deciles.mean_income.{d}", + f"Decile {d}: mean {spec.income_variable} per household", + "deciles", + "gbp_household", + year, + float(np.sum(hw[in_decile] * income[in_decile])) / weight + if weight + else float("nan"), + group="Own-decile levels", + ) + households[str(year)] = { + "household_id": array(sim, "household_id", year).astype(np.int64), + "household_weight": hw.astype(np.float64), + "household_count_people": people.astype(np.float64), + "decile": decile, + "income": income, + } + + +def _finite_metrics_check(rec: Recorder): + bad = [ + f"{m['id']} {fiscal_year_label(int(year))}" + for m in rec.metrics.values() + for year, value in m["values"].items() + if not np.isfinite(value) + ] + rec.check( + "finite:metrics", + "Every reported figure is finite", + not bad, + "; ".join(bad[:20]) + (f" and {len(bad) - 20} more" if len(bad) > 20 else ""), + section="finite", + ) + + +def weighted_total(sim, variable: str, year: int) -> float: + """The household-weighted total of a variable of any entity.""" + hw = array(sim, "household_weight", year).astype(np.float64) + values = array(sim, variable, year).astype(np.float64) + return float(np.sum(hw * to_household(sim, values, entity_of(sim, variable)))) + + +def as_standard_reforms(reforms) -> list: + """Accept parsed reforms or raw ``{name, parameters, ...}`` mappings.""" + reforms = list(reforms or []) + if all(isinstance(r, StandardReform) for r in reforms): + return reforms + return parse_reforms({"reforms": reforms}) + + +def measure_reforms( + factory: Callable, + baseline_parameters, + baseline_balance: Mapping[int, float], + baseline_targets: Mapping[int, Mapping[str, float]], + reforms: Sequence[StandardReform], + years: Sequence[int], + rec: Recorder, +) -> list: + """Cost each reform against the baseline run, and check it responds. + + Records each reform's change in gov_balance by year and, where the + reform declares what it must move, the change in that variable's total + and whether it moved enough in the right direction every year. + + First rebuilds an unreformed simulation the same way, which must + reproduce the baseline exactly: otherwise reform costings would include + differences that no reform caused. + """ + outcomes = [] + try: + fresh = factory(reform=None) + for year in years: + again = weighted_total(fresh, "gov_balance", year) + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + again == baseline_balance[year], + f"{fiscal_year_label(year)}: £{again:,.0f} against £{baseline_balance[year]:,.0f}", + section="determinism", + scope=str(year), + ) + del fresh + except Exception as error: # noqa: BLE001 + rec.check( + "determinism:rebuilt_baseline", + "A rebuilt unreformed simulation reproduces the baseline gov_balance exactly", + False, + f"{type(error).__name__}: {error}", + section="determinism", + ) + gc.collect() + for reform in reforms: + outcome = { + "id": reform.id, + "name": reform.name, + "parameters": None, + "responds": None, + "error": None, + } + target_changes = {} + try: + parameters = reform_parameters(reform, baseline_parameters, years) + outcome["parameters"] = parameters + reformed = factory(reform=parameters) + for year in years: + rec.metric( + f"reforms.{reform.id}", + reform.name, + "reforms", + "gbp", + year, + weighted_total(reformed, "gov_balance", year) + - baseline_balance[year], + ) + if reform.responds is not None: + variable = reform.responds.variable + change = ( + weighted_total(reformed, variable, year) + - baseline_targets[year][variable] + ) + target_changes[year] = change + rec.metric( + f"reform_targets.{reform.id}", + f"{reform.name}: change in {variable}", + "reform_targets", + "gbp", + year, + change, + ) + del reformed + except Exception as error: # noqa: BLE001 - one reform's failure is reported, not fatal + outcome["error"] = f"{type(error).__name__}: {error}" + if reform.responds is not None: + outcome["responds"] = { + "variable": reform.responds.variable, + "direction": reform.responds.direction, + "minimum": reform.responds.minimum, + } + if outcome["error"] is None: + ok, detail = responsiveness(reform.responds, target_changes) + else: + ok, detail = False, "the reform did not run" + rec.check( + f"responsive:{reform.id}", + f"'{reform.name}' moves {reform.responds.variable} " + f"{reform.responds.direction} by at least " + f"£{reform.responds.minimum / 1e9:,.2f}bn in every year", + ok, + detail, + section="reforms", + ) + gc.collect() + outcomes.append(outcome) + return outcomes + + +def measure( + sim, + factory: Callable, + partition: Partition, + spec: Spec, + *, + label: str, + reforms: Sequence[Mapping] = (), + run_guard: bool = True, + provenance: Optional[Mapping] = None, + guard_variables: Optional[Sequence[str]] = None, +) -> tuple: + """Measure a simulation over the spec's horizon. + + ``sim`` is a fresh simulation on the dataset; ``factory(reform=None)`` + builds another on the same dataset. Returns ``(result, households)``. + """ + started = time.time() + timings = {} + rec = Recorder() + model_lists, list_problems = read_model_lists(partition) + problems = list_problems + closure_problems( + partition, model_lists, sim.tax_benefit_system.variables + ) + rec.check( + "closure", + "Each model total's components are exactly the partition's heads and declared double counts", + not problems, + "; ".join(problems), + section="partition", + ) + snapshot = snapshot_inputs(sim, partition.component_variables()) + + reforms = as_standard_reforms(reforms) + target_variables = sorted({r.responds.variable for r in reforms if r.responds}) + households = {} + baseline_balance, baseline_targets = {}, {} + reset = getattr(sim, "_invalidate_all_caches", None) + for year in spec.years: + t = time.time() + rec.scope = str(year) + measure_year(sim, partition, spec, year, rec, households) + rec.scope = "" + baseline_balance[year] = weighted_total(sim, "gov_balance", year) + baseline_targets[year] = { + variable: weighted_total(sim, variable, year) + for variable in target_variables + } + timings[f"year_{year}"] = round(time.time() - t, 2) + if reset is not None: + # Keep memory flat across the horizon; supplied inputs survive. + reset() + gc.collect() + + guard = {"status": "skipped", "reason": "not requested"} + if run_guard: + t = time.time() + guard = counted_once( + sim, + partition, + spec.guard_year, + spec.counted_once_tolerance, + snapshot, + active=active_parameters(sim, partition, spec.guard_year), + variables_to_check=guard_variables, + ) + timings["counted_once"] = round(time.time() - t, 2) + for result in guard.get("components", []): + rec.check( + f"counted_once:{result['variable']}", + f"Raising {result['variable']} by £1 raises its own head by £1, moves no " + "other tax head and raises no other benefit", + None if result["status"] == "skipped" else result["status"] == "pass", + result["detail"], + section="counted_once", + ) + for result in guard.get("double_counts", []): + rec.check( + f"double_count_present:{result['variable']}", + f"{result['variable']} is still contained in {result['contained_in']}", + result["status"] == "pass", + result["detail"], + section="counted_once", + ) + if "reset_reproduces_baseline" in guard: + rec.check( + "counted_once:reset_reproduces_baseline", + "The counted-once check leaves the baseline unchanged", + bool(guard["reset_reproduces_baseline"]), + "recomputed baseline totals differ from the first run", + section="counted_once", + ) + rec.check( + "counted_once:ran", + "The counted-once check ran", + guard.get("status") == "ran", + guard.get("reason", ""), + severity="warn", + section="counted_once", + ) + + t = time.time() + reform_outcomes = measure_reforms( + factory, + sim.tax_benefit_system.parameters, + baseline_balance, + baseline_targets, + reforms, + spec.years, + rec, + ) + timings["reforms"] = round(time.time() - t, 2) + for outcome in reform_outcomes: + rec.check( + f"reform_runs:{outcome['id']}", + f"The reform '{outcome['name']}' runs", + outcome["error"] is None, + outcome["error"] or "", + section="reforms", + ) + # After the reform costings, so that they are covered too. + _finite_metrics_check(rec) + timings["total"] = round(time.time() - started, 2) + + provenance = dict(provenance or {}) + provenance["tool_sha256"] = tool_digest() + provenance["definitions_sha256"] = definitions_digest(spec, reforms) + provenance["metrics_version"] = ( + f"{TOOL_SCHEMA}+{provenance['tool_sha256'][:16]}" + f"+{provenance['definitions_sha256'][:16]}" + ) + result = { + "schema": TOOL_SCHEMA, + "status": "ok", + "label": label, + "years": spec.years, + "provenance": provenance, + "runtime": runtime_identity(), + "partition": partition.as_dict(), + "metrics": list(rec.metrics.values()), + "checks": list(rec.checks.values()), + "guard": guard, + "reforms": reform_outcomes, + "timings": timings, + } + return result, households + + +PRIVATE_DIRECTORY = "private" +HOUSEHOLDS_FILE = "households.npz" + + +def write_result(out_dir, result: Mapping, households: Optional[Mapping] = None): + """Write the aggregates to result.json and household detail to private/. + + ``result.json`` holds only aggregates and may be published. Everything + under ``private/`` is household-level detail from licensed microdata: it + exists for the decile comparison and must never be uploaded or shared. + """ + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + with open(out_dir / "result.json", "w") as f: + json.dump(result, f, indent=1, sort_keys=False, allow_nan=True) + if households: + arrays = { + f"{year}__{name}": values + for year, columns in households.items() + for name, values in columns.items() + } + private = out_dir / PRIVATE_DIRECTORY + private.mkdir(exist_ok=True) + np.savez_compressed(private / HOUSEHOLDS_FILE, **arrays) + + +def read_result(out_dir) -> tuple: + """Load ``(result, households)`` written by :func:`write_result`.""" + out_dir = Path(out_dir) + path = out_dir / "result.json" + if not path.exists(): + return None, None + with open(path) as f: + result = json.load(f) + households = None + npz = out_dir / PRIVATE_DIRECTORY / HOUSEHOLDS_FILE + if npz.exists(): + households = {} + with np.load(npz) as data: + for key in data.files: + year, _, name = key.partition("__") + households.setdefault(year, {})[name] = data[key] + return result, households diff --git a/policyengine_uk/utils/aggregate_impact/partition.py b/policyengine_uk/utils/aggregate_impact/partition.py new file mode 100644 index 0000000000..a853270b53 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/partition.py @@ -0,0 +1,545 @@ +"""The fiscal partition: loading, structural validation and closure checks. + +The partition lives with the model at +``policyengine_uk/data/fiscal_partition.yaml`` (schema 1; its header defines +the fields). This module reads the YAML directly, not through any loader in +the model, so the pull request head's copy of this tool can read the base +checkout's partition. +""" + +from __future__ import annotations + +import hashlib +import importlib +import importlib.util +from collections import Counter +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable, Mapping, Optional + +import yaml + +SUPPORTED_SCHEMA_VERSIONS = {1} +PARTITION_RELATIVE_PATH = Path("data") / "fiscal_partition.yaml" +ENTITIES = {"person", "benunit", "household"} +SIGNS = {None, "nonnegative", "any"} + +# Where each total's component list lives in the model, and which kind of +# heads it divides into. A total in the file may override these. +KNOWN_TOTALS = { + "gov_tax": ("tax", "policyengine_uk.variables.gov.gov_tax", "GOV_TAX_VARIABLES"), + "household_tax": ( + "tax", + "policyengine_uk.variables.gov.hmrc.household_tax", + "HOUSEHOLD_TAX_VARIABLES", + ), + "pre_budget_change_household_tax": ( + "tax", + "policyengine_uk.variables.contrib.policyengine.pre_budget_change_household_tax", + "PRE_BUDGET_CHANGE_HOUSEHOLD_TAX_VARIABLES", + ), + "gov_spending": ( + "spending", + "policyengine_uk.variables.gov.gov_spending", + "GOV_SPENDING_VARIABLES", + ), + "household_benefits": ( + "spending", + "policyengine_uk.variables.household.income.household_benefits", + "HOUSEHOLD_BENEFIT_VARIABLES", + ), +} +GOVERNMENT_TOTALS = {"gov_tax", "gov_spending"} + + +class PartitionError(ValueError): + """The partition file is malformed or internally inconsistent.""" + + +@dataclass(frozen=True) +class TotalSpec: + name: str + kind: str # "tax" or "spending" + module: str + list_name: str + label: str = "" + dropped_when: tuple = () # ((parameter path, (variable, ...)), ...) + + +@dataclass(frozen=True) +class Head: + id: str + label: str + variable: str + totals: tuple + report_group: str + kind: str # "tax" or "spending" + entity: Optional[str] = None + obr_line: Optional[str] = None + payer: Optional[str] = None + sign: Optional[str] = None + zero_at_baseline: bool = False + caseload: bool = False + reads: tuple = () + notes: tuple = () + + +@dataclass(frozen=True) +class DoubleCount: + variable: str + contained_in: str + totals: tuple + issue: str = "" + + +@dataclass(frozen=True) +class Exclusion: + variable: str + kind: str = "" + head: str = "" + spans: tuple = () + note: str = "" + + +@dataclass(frozen=True) +class Partition: + schema_version: int + totals: Mapping[str, TotalSpec] + tax_heads: tuple + spending_heads: tuple + double_counts: tuple + exclusions: tuple + overlap_guards: tuple = () + source: str = "" + sha256: str = "" + raw: Mapping = field(default_factory=dict, compare=False, repr=False) + + @property + def heads(self) -> tuple: + return self.tax_heads + self.spending_heads + + def dropped(self, total: str, active: Iterable[str] = ()) -> set: + """Variables the total leaves out while the given parameters are true.""" + active = set(active) + spec = self.totals.get(total) + if spec is None: + return set() + return { + v + for parameter, variables in spec.dropped_when + if parameter in active + for v in variables + } + + def heads_in(self, total: str, active: Iterable[str] = ()) -> list: + dropped = self.dropped(total, active) + return [ + h for h in self.heads if total in h.totals and h.variable not in dropped + ] + + def double_counts_in(self, total: str, active: Iterable[str] = ()) -> list: + dropped = self.dropped(total, active) + return [ + d + for d in self.double_counts + if total in d.totals and d.variable not in dropped + ] + + def components(self, total: str, active: Iterable[str] = ()) -> list: + """The variables the model should add for ``total``, per the partition.""" + return [h.variable for h in self.heads_in(total, active)] + [ + d.variable for d in self.double_counts_in(total, active) + ] + + def component_variables(self) -> list: + """Every variable named as a head or a declared double count, once.""" + return list( + dict.fromkeys( + [h.variable for h in self.heads] + + [d.variable for d in self.double_counts] + ) + ) + + def head_by_variable(self, variable: str) -> Optional[Head]: + for head in self.heads: + if head.variable == variable: + return head + return None + + def is_tax_total(self, total: str) -> bool: + spec = self.totals.get(total) + return spec is not None and spec.kind == "tax" + + def dropped_when_parameters(self) -> list: + return list( + dict.fromkeys( + p for spec in self.totals.values() for p, _ in spec.dropped_when + ) + ) + + def as_dict(self) -> dict: + return { + "source": self.source, + "sha256": self.sha256, + "schema_version": self.schema_version, + "tax_heads": [_head_dict(h) for h in self.tax_heads], + "spending_heads": [_head_dict(h) for h in self.spending_heads], + "model_double_counts": [ + { + "variable": d.variable, + "contained_in": d.contained_in, + "totals": list(d.totals), + "issue": d.issue, + } + for d in self.double_counts + ], + "totals": { + name: { + "kind": t.kind, + "module": t.module, + "list": t.list_name, + "dropped_when": {p: list(v) for p, v in t.dropped_when}, + } + for name, t in self.totals.items() + }, + } + + +def _head_dict(head: Head) -> dict: + return { + "id": head.id, + "label": head.label, + "variable": head.variable, + "totals": list(head.totals), + "report_group": head.report_group, + "kind": head.kind, + "entity": head.entity, + "obr_line": head.obr_line, + "payer": head.payer, + "sign": head.sign, + "zero_at_baseline": head.zero_at_baseline, + "caseload": head.caseload, + "notes": list(head.notes), + } + + +def group_label(report_group: str) -> str: + """``excise_duties`` -> ``Excise duties``; labels already in prose pass through.""" + text = report_group.replace("_", " ").strip() + return text[:1].upper() + text[1:] if text else "Other" + + +def default_partition_path() -> Optional[Path]: + """The partition shipped with the installed ``policyengine_uk``, if any.""" + spec = importlib.util.find_spec("policyengine_uk") + if spec is None or not spec.submodule_search_locations: + return None + for location in spec.submodule_search_locations: + candidate = Path(location) / PARTITION_RELATIVE_PATH + if candidate.exists(): + return candidate + return None + + +def _as_tuple(value, field_name: str, where: str) -> tuple: + if value is None: + return () + if isinstance(value, str): + raise PartitionError(f"{where}: {field_name} must be a list, not a string") + return tuple(str(v) for v in value) + + +def _parse_total(name: str, raw: Mapping) -> TotalSpec: + if raw is None: + raw = {} + if not isinstance(raw, Mapping): + raise PartitionError(f"totals.{name} must be a mapping") + known = KNOWN_TOTALS.get(name) + kind = raw.get("kind") or (known[0] if known else None) + module = raw.get("module") or (known[1] if known else None) + list_name = raw.get("list") or (known[2] if known else None) + if kind not in ("tax", "spending") or not module or not list_name: + raise PartitionError( + f"totals.{name}: an unfamiliar total needs kind (tax or spending), module and list" + ) + dropped = [] + for parameter, variables in (raw.get("dropped_when") or {}).items(): + dropped.append( + (str(parameter), _as_tuple(variables, "dropped_when", f"totals.{name}")) + ) + return TotalSpec( + name=name, + kind=kind, + module=str(module), + list_name=str(list_name), + label=str(raw.get("label") or ""), + dropped_when=tuple(dropped), + ) + + +def _parse_head(raw: Mapping, kind: str, index: int) -> Head: + where = f"{kind}_heads[{index}]" + if not isinstance(raw, Mapping): + raise PartitionError(f"{where}: must be a mapping") + missing = [k for k in ("id", "label", "variable", "totals") if not raw.get(k)] + if missing: + raise PartitionError(f"{where}: missing {missing}") + zero = raw.get("zero_at_baseline") + if zero not in (None, "expected"): + raise PartitionError(f"{where}: zero_at_baseline must be 'expected' or absent") + notes = tuple( + f"{key.replace('_', ' ')}: {' '.join(str(raw[key]).split())}" + for key in ("obr_line_note", "zero_at_baseline_note", "data_note", "model_note") + if raw.get(key) + ) + reads = tuple( + str(r.get("head")) for r in (raw.get("reads") or []) if isinstance(r, Mapping) + ) + caseload = raw.get("caseload") + return Head( + id=str(raw["id"]), + label=str(raw["label"]), + variable=str(raw["variable"]), + totals=_as_tuple(raw.get("totals"), "totals", where), + report_group=str(raw.get("report_group") or "other"), + kind=kind, + entity=raw.get("entity"), + obr_line=raw.get("obr_line"), + payer=raw.get("payer"), + sign=raw.get("sign"), + zero_at_baseline=zero == "expected", + caseload=bool(caseload) if caseload is not None else kind == "spending", + reads=reads, + notes=notes, + ) + + +def parse_partition( + raw: Mapping, source: str = "", sha256: str = "" +) -> Partition: + """Build a partition from parsed YAML, raising on any inconsistency.""" + if not isinstance(raw, Mapping): + raise PartitionError(f"{source}: the partition must be a mapping") + version = raw.get("schema_version") + if version not in SUPPORTED_SCHEMA_VERSIONS: + raise PartitionError( + f"{source}: schema_version {version!r} is not one of " + f"{sorted(SUPPORTED_SCHEMA_VERSIONS)}" + ) + totals = { + str(name): _parse_total(str(name), spec) + for name, spec in (raw.get("totals") or {}).items() + } + tax_heads = tuple( + _parse_head(h, "tax", i) for i, h in enumerate(raw.get("tax_heads") or []) + ) + spending_heads = tuple( + _parse_head(h, "spending", i) + for i, h in enumerate(raw.get("spending_heads") or []) + ) + double_counts = [] + for i, dc in enumerate(raw.get("model_double_counts") or []): + where = f"model_double_counts[{i}]" + if ( + not isinstance(dc, Mapping) + or not dc.get("variable") + or not dc.get("contained_in") + ): + raise PartitionError(f"{source}: {where} needs variable and contained_in") + double_counts.append( + DoubleCount( + variable=str(dc["variable"]), + contained_in=str(dc["contained_in"]), + totals=_as_tuple(dc.get("totals"), "totals", where), + issue=str(dc.get("issue") or ""), + ) + ) + exclusions = tuple( + Exclusion( + variable=str(e.get("variable")), + kind=str(e.get("kind") or ""), + head=str(e.get("head") or ""), + spans=_as_tuple(e.get("spans"), "spans", "exclusions"), + note=str(e.get("note") or e.get("reason") or ""), + ) + for e in (raw.get("exclusions") or []) + if isinstance(e, Mapping) + ) + partition = Partition( + schema_version=version, + totals=totals, + tax_heads=tax_heads, + spending_heads=spending_heads, + double_counts=tuple(double_counts), + exclusions=exclusions, + overlap_guards=tuple(raw.get("overlap_guards") or ()), + source=source, + sha256=sha256, + raw=raw, + ) + problems = structural_problems(partition) + if problems: + raise PartitionError(f"{source}:\n- " + "\n- ".join(problems)) + return partition + + +def load_partition(path) -> Partition: + path = Path(path) + data = path.read_bytes() + return parse_partition( + yaml.safe_load(data), source=str(path), sha256=hashlib.sha256(data).hexdigest() + ) + + +def structural_problems(partition: Partition) -> list: + """Inconsistencies visible from the partition alone, without the model.""" + problems = [] + heads = partition.heads + head_ids = {h.id for h in heads} + for name, count in Counter(h.id for h in heads).items(): + if count > 1: + problems.append(f"head id {name!r} is used {count} times") + for name, count in Counter(h.variable for h in heads).items(): + if count > 1: + problems.append(f"variable {name!r} is the variable of {count} heads") + head_variables = {h.variable for h in heads} + for head in heads: + if len(set(head.totals)) != len(head.totals): + problems.append(f"head {head.id!r} lists a total twice") + for total in head.totals: + spec = partition.totals.get(total) + if spec is None: + problems.append(f"head {head.id!r} names unknown total {total!r}") + elif spec.kind != head.kind: + problems.append( + f"{head.kind} head {head.id!r} is in {total!r}, a {spec.kind} total" + ) + if head.entity is not None and head.entity not in ENTITIES: + problems.append(f"head {head.id!r} has unknown entity {head.entity!r}") + if head.sign not in SIGNS: + problems.append(f"head {head.id!r} has unknown sign {head.sign!r}") + for read in head.reads: + if read not in head_ids: + problems.append( + f"head {head.id!r} declares a read of unknown head {read!r}" + ) + for name, spec in partition.totals.items(): + listed = set(partition.components(name)) + for parameter, variables in spec.dropped_when: + for variable in variables: + if variable not in listed: + problems.append( + f"totals.{name} drops {variable!r} under {parameter}, but no head " + "or double count of that total is that variable" + ) + for dc in partition.double_counts: + if dc.variable in head_variables: + problems.append( + f"double count {dc.variable!r} is also a head; it would be counted twice" + ) + container = partition.head_by_variable(dc.contained_in) + if container is None: + problems.append( + f"double count {dc.variable!r} is contained in {dc.contained_in!r}, " + "which is not a head variable" + ) + else: + missing = [t for t in dc.totals if t not in container.totals] + if missing: + problems.append( + f"double count {dc.variable!r} is in {missing}, which its " + f"container {dc.contained_in!r} is not" + ) + for total in dc.totals: + if total not in partition.totals: + problems.append( + f"double count {dc.variable!r} names unknown total {total!r}" + ) + for name, count in Counter(d.variable for d in partition.double_counts).items(): + if count > 1: + problems.append(f"double count {name!r} is declared {count} times") + for exclusion in partition.exclusions: + if exclusion.variable in head_variables: + problems.append( + f"excluded variable {exclusion.variable!r} is also a head variable" + ) + for referenced in ([exclusion.head] if exclusion.head else []) + list( + exclusion.spans + ): + if referenced not in head_ids: + problems.append( + f"exclusion {exclusion.variable!r} refers to unknown head {referenced!r}" + ) + return problems + + +def read_model_lists(partition: Partition) -> tuple: + """Import each total's component list from the model. + + Returns ``(lists, problems)``: the lists that could be read, keyed by + total, and a description of every total whose list could not be read. + """ + lists, problems = {}, [] + for name, spec in partition.totals.items(): + try: + module = importlib.import_module(spec.module) + values = getattr(module, spec.list_name) + except (ImportError, AttributeError) as error: + problems.append( + f"{name}: cannot read {spec.module}.{spec.list_name}: {error}" + ) + continue + lists[name] = [str(v) for v in values] + return lists, problems + + +def closure_problems( + partition: Partition, + model_lists: Mapping[str, Iterable[str]], + model_variables: Optional[Mapping] = None, +) -> list: + """Differences between the partition and the model's own component lists. + + For every total, the heads and double counts naming it must be exactly + the model's list, each listed once; every named variable must exist, and + a head's declared entity must be its variable's entity. + """ + problems = [] + for total, listed in model_lists.items(): + listed = list(listed) + for variable, count in Counter(listed).items(): + if count > 1: + problems.append(f"{total}: the model lists {variable!r} {count} times") + expected, assigned = set(listed), set(partition.components(total)) + for variable in sorted(expected - assigned): + problems.append( + f"{total}: the model adds {variable!r}, which no head or declared " + "double count covers" + ) + for variable in sorted(assigned - expected): + problems.append( + f"{total}: the partition puts {variable!r} in this total, but the " + "model does not add it" + ) + if model_variables is not None: + named = partition.component_variables() + [ + d.contained_in for d in partition.double_counts + ] + for variable in sorted(set(named) - set(model_variables)): + problems.append(f"{variable!r} is not a model variable") + for head in partition.heads: + definition = ( + model_variables.get(head.variable) + if hasattr(model_variables, "get") + else None + ) + if ( + head.entity + and definition is not None + and definition.entity.key != head.entity + ): + problems.append( + f"head {head.id!r} says entity {head.entity!r}, but {head.variable} " + f"is a {definition.entity.key} variable" + ) + return problems diff --git a/policyengine_uk/utils/aggregate_impact/reforms.py b/policyengine_uk/utils/aggregate_impact/reforms.py new file mode 100644 index 0000000000..c5ec5a77b8 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/reforms.py @@ -0,0 +1,148 @@ +"""The standard reform set: loading, building reforms, and responsiveness.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping, Optional, Sequence + +import yaml + +from .spec import fiscal_year_label + +DEFAULT_REFORMS_PATH = Path(__file__).with_name("standard_reforms.yaml") +_INDEXED = re.compile(r"^(?P[^\[\]]+)\[(?P\d+)\]$") + + +class ReformSpecError(ValueError): + pass + + +@dataclass(frozen=True) +class Responds: + """What a reform must do to the weighted total of one model variable.""" + + variable: str + direction: str # "up" or "down" + minimum: float + + +@dataclass(frozen=True) +class StandardReform: + id: str + name: str + parameters: Mapping = field(default_factory=dict) + scale: Mapping = field(default_factory=dict) + responds: Optional[Responds] = None + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +def parse_reforms(raw: Mapping) -> list: + """Parse a reform file. + + Accepts the standard schema and the reform-cost fixture schema (a + ``reforms`` list of ``name`` and ``parameters``, with no ``responds``). + """ + reforms, seen = [], set() + for index, entry in enumerate((raw or {}).get("reforms") or []): + where = f"reforms[{index}]" + if not isinstance(entry, Mapping) or not entry.get("name"): + raise ReformSpecError(f"{where}: needs a name") + parameters = dict(entry.get("parameters") or {}) + scale = {str(k): float(v) for k, v in (entry.get("scale") or {}).items()} + if not parameters and not scale: + raise ReformSpecError(f"{where}: needs parameters or scale") + for path, factor in scale.items(): + if not factor > 0: + raise ReformSpecError( + f"{where}: scale factor for {path} must be positive" + ) + responds = None + if entry.get("responds"): + r = entry["responds"] + if r.get("direction") not in ("up", "down"): + raise ReformSpecError(f"{where}: responds.direction must be up or down") + minimum = float(r.get("minimum", 0)) + if not r.get("variable") or not minimum > 0: + raise ReformSpecError( + f"{where}: responds needs a variable and a positive minimum" + ) + responds = Responds(str(r["variable"]), r["direction"], minimum) + reform_id = str(entry.get("id") or _slug(str(entry["name"]))) + if reform_id in seen: + raise ReformSpecError(f"{where}: reform id {reform_id!r} is used twice") + seen.add(reform_id) + reforms.append( + StandardReform(reform_id, str(entry["name"]), parameters, scale, responds) + ) + return reforms + + +def load_reforms(path=None) -> list: + with open(path or DEFAULT_REFORMS_PATH) as f: + return parse_reforms(yaml.safe_load(f)) + + +def parameter_node(root, path: str): + """Navigate ``a.b[0].c`` from a parameter tree's root.""" + node = root + for part in path.split("."): + indexed = _INDEXED.match(part) + if indexed: + node = getattr(node, indexed["name"]).brackets[int(indexed["index"])] + else: + node = getattr(node, part) + return node + + +def reform_parameters( + reform: StandardReform, baseline_parameters, years: Sequence[int] +) -> dict: + """The parameter-change dict for ``Microsimulation(reform=...)``. + + A scaled parameter gets one bare-year entry per horizon year: the + baseline's value for that model year times the factor. On this route a + bare year sets that model year alone. + """ + changes = dict(reform.parameters) + for path, factor in reform.scale.items(): + node = parameter_node(baseline_parameters, path) + changes[path] = { + str(year): float(node(f"{year}-01-01")) * factor for year in years + } + return changes + + +def responsiveness(responds: Responds, changes: Mapping[int, float]) -> tuple: + """Whether a reform moved its target enough, the right way, in every year. + + ``changes`` maps year to the reformed total minus the baseline total. + Returns ``(ok, detail)``; ``detail`` names the failing years. + """ + sign = 1.0 if responds.direction == "up" else -1.0 + failing = [] + for year, change in sorted(changes.items()): + moved = sign * change + if not moved >= responds.minimum: + if change == 0: + what = "did not move" + elif moved < 0: + what = f"moved the wrong way (£{change:,.0f})" + else: + what = f"moved only £{change:,.0f}" + failing.append( + f"{fiscal_year_label(int(year))}: {responds.variable} {what}" + ) + if not changes: + failing.append("no year was measured") + detail = "" + if failing: + detail = ( + f"expected {responds.variable} {responds.direction} by at least " + f"£{responds.minimum:,.0f} a year; " + "; ".join(failing) + ) + return not failing, detail diff --git a/policyengine_uk/utils/aggregate_impact/render.py b/policyengine_uk/utils/aggregate_impact/render.py new file mode 100644 index 0000000000..00830fd275 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/render.py @@ -0,0 +1,410 @@ +"""Render a comparison as a Markdown pull request comment.""" + +from __future__ import annotations + +import math +from typing import Mapping, Optional + +from . import COMMENT_MARKER +from .spec import Spec, fiscal_year_label + +SECTIONS = ( + ("receipts", "Receipts by tax head"), + ("spending", "Spending by benefit and service"), + ("caseloads", "Caseloads"), + ("balance", "Government balance, each component counted once"), + ("model_totals", "Model totals"), + ("double_counts", "Components the model counts twice"), + ("household_totals", "Household income totals"), + ("population", "Population"), + ("poverty", "Poverty rates"), + ("poverty_count", "People in poverty"), + ("inequality", "Inequality"), + ("deciles", "Mean household net income in each side's own deciles"), + ("reforms", "Standard reform costings (change in gov_balance)"), + ("reform_targets", "Standard reforms: change in each reform's target"), +) +MAX_FLAG_ROWS = 40 + + +def _finite(value) -> bool: + return isinstance(value, (int, float)) and math.isfinite(value) + + +def fmt_level(value, kind) -> str: + if not _finite(value): + return "n/a" + if kind == "gbp": + return f"£{value / 1e9:,.2f}bn" + if kind == "gbp_household": + return f"£{value:,.0f}" + if kind == "count": + return f"{value / 1e3:,.0f}k" + if kind == "rate": + return f"{100 * value:.1f}%" + return f"{value:.4f}" + + +def fmt_change(value, kind) -> str: + if value is None or not _finite(value): + return "n/a" + if value == 0: + return "0" + sign = "+" if value > 0 else "-" + size = abs(value) + if kind == "gbp": + if size >= 1e9: + return f"{sign}£{size / 1e9:,.2f}bn" + if size >= 1e5: + return f"{sign}£{size / 1e6:,.1f}m" + return f"{sign}£{size:,.0f}" + if kind == "gbp_household": + return f"{sign}£{size:,.2f}" if size < 10 else f"{sign}£{size:,.0f}" + if kind == "count": + if size >= 1e3: + return f"{sign}{size / 1e3:,.1f}k" + return f"{sign}{size:,.0f}" + if kind == "rate": + return f"{sign}{100 * size:.2f}pp" + return f"{sign}{size:.4f}" + + +def _table(header, rows) -> str: + lines = [ + "| " + " | ".join(header) + " |", + "|" + "|".join(["---"] + ["---:"] * (len(header) - 1)) + "|", + ] + lines += ["| " + " | ".join(str(c) for c in row) + " |" for row in rows] + return "\n".join(lines) + + +def _year_header(years): + return [fiscal_year_label(int(y)) for y in years] + + +def _section_rows(summary, section): + return [r for r in summary["rows"] if r["section"] == section] + + +def _changes_table(rows, years) -> str: + changed = [r for r in rows if r["changed"]] + if not changed: + return "No change in any year." + body = [] + for row in changed: + if row["only_in"] == "head": + cells = [ + fmt_level(row["values"][str(y)]["head"], row["kind"]) for y in years + ] + body.append([f"{row['label']} (new: head levels)"] + cells) + elif row["only_in"] == "base": + cells = [ + fmt_level(row["values"][str(y)]["base"], row["kind"]) for y in years + ] + body.append([f"{row['label']} (removed: base levels)"] + cells) + else: + body.append( + [row["label"]] + + [ + fmt_change(row["values"][str(y)]["delta"], row["kind"]) + for y in years + ] + ) + return _table(["Measure"] + _year_header(years), body) + + +def _levels_table(rows, years) -> str: + body = [ + [row["label"]] + + [fmt_level(row["values"][str(y)]["head"], row["kind"]) for y in years] + for row in rows + ] + return _table(["Measure (head)"] + _year_header(years), body) + + +def _details(summary_text, body) -> str: + return f"
\n{summary_text}\n\n{body}\n\n
" + + +def _deciles(summary, years) -> str: + deciles = summary.get("deciles") or {} + if not deciles: + return "Not available: both sides need household detail." + present = [y for y in years if str(y) in deciles] + if all(row["mean_change"] == 0 for y in present for row in deciles[str(y)]): + return "No household's net income changes in any year." + labels = [row["decile"] for row in deciles[str(present[0])]] + body = [] + for index, decile in enumerate(labels): + cells = [] + for year in present: + row = deciles[str(year)][index] + if row["mean_change"] == 0 or not _finite(row["mean_change"]): + cells.append("0" if row["mean_change"] == 0 else "n/a") + else: + cells.append( + f"{fmt_change(row['mean_change'], 'gbp')} ({100 * row['relative_change']:+.2f}%)" + ) + body.append([decile if decile == "all" else f"Decile {decile}"] + cells) + text = ( + "Mean change in household net income per household, and as a share of the " + "decile's base income; households stay in the base run's deciles.\n\n" + + _table(["Base decile"] + _year_header(present), body) + ) + winners = [] + for year in present: + overall = deciles[str(year)][-1] + winners.append( + [ + fiscal_year_label(year), + f"{100 * overall['share_gaining']:.1f}%", + f"{100 * overall['share_losing']:.1f}%", + ] + ) + text += "\n\n" + _table( + ["Year", "People gaining over £1", "People losing over £1"], winners + ) + if summary.get("decile_notes"): + text += "\n\n" + "\n".join(f"- {note}" for note in summary["decile_notes"]) + return text + + +def _flags(summary) -> str: + flags = list(summary.get("flags") or []) + flags += [ + { + "label": f"Decile {f['decile']} household net income, relative", + "year": f["year"], + "kind": "relative", + "base": None, + "head": None, + "delta": f["relative_change"], + "limit": None, + "section": "deciles", + } + for f in summary.get("decile_flags") or [] + ] + if not flags: + return "None." + flags.sort(key=lambda f: -(abs(f["delta"]) / f["limit"]) if f.get("limit") else 0) + shown = flags[:MAX_FLAG_ROWS] + body = [] + for f in shown: + if f["kind"] == "relative": + body.append( + [ + f["label"], + fiscal_year_label(f["year"]), + "", + "", + f"{100 * f['delta']:+.2f}%", + ] + ) + else: + body.append( + [ + f["label"], + fiscal_year_label(f["year"]), + fmt_level(f["base"], f["kind"]), + fmt_level(f["head"], f["kind"]), + fmt_change(f["delta"], f["kind"]), + ] + ) + text = _table(["Measure", "Year", "Base", "Head", "Change"], body) + if len(flags) > len(shown): + text += f"\n\n{len(flags) - len(shown)} more flags are in the report artifact." + return text + + +def _check_list(checks, classification) -> list: + out = [] + for check in checks: + if check["classification"] != classification: + continue + details = check["head_details"] or check["base_details"] + detail = f": {'; '.join(details[:3])}" if details else "" + out.append(f"- {check['label']} (`{check['id']}`){detail}") + return out + + +def _guard(summary) -> str: + guard = (summary.get("head") or {}).get("guard") or {} + if guard.get("status") != "ran": + return f"Counted-once check not run: {guard.get('reason', 'unknown reason')}." + tolerance = guard.get("tolerance", 1e-3) + tax_totals = {"gov_tax", "household_tax", "pre_budget_change_household_tax"} + components = guard.get("components", []) + tested = [c for c in components if c["status"] != "skipped"] + failed = [c for c in tested if c["status"] == "fail"] + lines = [ + f"Each of {len(tested)} components was raised by £1 per unit in " + f"{fiscal_year_label(guard['year'])}: tax totals must rise by exactly £1 " + "per unit and spending totals by at most £1. " + f"{len(failed)} failed; {len(components) - len(tested)} were skipped " + "(zero for every unit, or not money)." + ] + for c in failed: + moves = ", ".join(f"`{t}` {k:.4f}" for t, k in c["coefficients"].items()) + lines.append(f"- `{c['variable']}`: {moves}") + + def violates(total, k): + return abs(k - 1) > tolerance if total in tax_totals else k > 1 + tolerance + + model = [ + (c["variable"], total, k) + for c in tested + for total, k in c.get("model_coefficients", {}).items() + if violates(total, k) + ] + if model: + lines += ["", "In the model's own totals:"] + lines += [f"- `{v}` moves `{t}` by {k:.4f} per unit" for v, t, k in model] + for dc in guard.get("double_counts", []): + state = "still contained" if dc["status"] == "pass" else "NO LONGER contained" + lines.append( + f"- declared double count `{dc['variable']}` is {state} in " + f"`{dc['contained_in']}` (coefficient {dc['coefficient']:.3f}; {dc['issue']})" + ) + return "\n".join(lines) + + +def _provenance(summary) -> str: + rows = [] + for side in ("base", "head"): + info = summary.get(side) + if not info: + rows.append([side, "no result", "", "", "", ""]) + continue + provenance = info.get("provenance") or {} + runtime = info.get("runtime") or {} + rows.append( + [ + side, + f"{provenance.get('git_ref', '')} `{str(provenance.get('git_sha', ''))[:10]}`", + f"uk {runtime.get('policyengine_uk')}, core {runtime.get('policyengine_core')}, py {runtime.get('python')} {runtime.get('machine', '')}", + f"`{str(runtime.get('distributions_sha256', ''))[:12]}`", + f"`{str((info.get('partition') or {}).get('sha256', ''))[:12]}`", + f"{(info.get('timings') or {}).get('total', '')}s", + ] + ) + dataset = ((summary.get("head") or {}).get("provenance") or {}).get("dataset") or {} + text = _table( + ["Side", "Commit", "Runtime", "Environment hash", "Partition hash", "Time"], + rows, + ) + if dataset: + text += ( + f"\n\nDataset: `{dataset.get('uri')}`, sha256 `{dataset.get('verified_sha256')}` " + f"(verified on both sides), {dataset.get('data_package')} {dataset.get('data_version')}." + ) + if summary.get("runtime_differences"): + text += "\n\nRuntime differences:\n" + "\n".join( + f"- {d}" for d in summary["runtime_differences"] + ) + return text + + +def render(summary: Mapping, spec: Spec, context: Optional[Mapping] = None) -> str: + """The full comment; sections are dropped from the end to fit GitHub's limit.""" + context = context or {} + if summary["status"] == "skipped": + notes = "\n".join(f"- {w}" for w in summary.get("warnings") or []) + return f"{COMMENT_MARKER}\n## Aggregate impact on certified data: skipped\n\n{notes}\n" + years = summary.get("years") or spec.years + regressions = [c for c in summary["checks"] if c["classification"] == "regression"] + pre_existing = [ + c for c in summary["checks"] if c["classification"] == "pre_existing" + ] + new_warnings = [ + c for c in summary["checks"] if c["classification"] == "new_warning" + ] + fixed = [c for c in summary["checks"] if c["classification"] == "fixed"] + flag_count = len(summary.get("flags") or []) + len( + summary.get("decile_flags") or [] + ) + verdict = "FAIL" if summary["status"] == "fail" else "pass" + + head_prov = (summary.get("head") or {}).get("provenance") or {} + base_prov = (summary.get("base") or {}).get("provenance") or {} + dataset = head_prov.get("dataset") or {} + parts = [ + COMMENT_MARKER, + f"## Aggregate impact on certified data: {verdict}", + "", + ( + f"Base `{str(base_prov.get('git_sha', '?'))[:10]}` against head " + f"`{str(head_prov.get('git_sha', '?'))[:10]}`, both on `{dataset.get('path', '?')}` " + f"({dataset.get('data_package', '')} {dataset.get('data_version', '')}, sha256 " + f"`{str(dataset.get('verified_sha256', '?'))[:12]}`), fiscal years " + f"{fiscal_year_label(years[0])} to {fiscal_year_label(years[-1])}. Each year is a full " + "microsimulation run on each side." + ), + "", + ( + f"**Checks:** {len(regressions)} broken by this change, {len(pre_existing)} failing on " + f"both sides, {len(fixed)} fixed. **Moves flagged for review:** {flag_count} " + "(thresholds proposed in `report_spec.yaml`, pending cos d1344; a flag does not fail the job)." + ), + ] + if context.get("run_url"): + parts.append(f"\n[Workflow run and full report]({context['run_url']})") + if summary["failures"]: + parts += ["", "### Failures", ""] + [f"- {f}" for f in summary["failures"]] + if new_warnings: + parts += ["", "### New warnings", ""] + _check_list( + summary["checks"], "new_warning" + ) + if summary["warnings"]: + parts += ["", "### Notes", ""] + [f"- {w}" for w in summary["warnings"]] + if fixed: + parts += ["", "### Fixed by this change", ""] + _check_list( + summary["checks"], "fixed" + ) + parts += ["", "### Moves flagged for review", "", _flags(summary)] + + unchanged = [] + for section, title in SECTIONS: + rows = _section_rows(summary, section) + if not rows: + continue + if not any(row["changed"] for row in rows): + unchanged.append(title[:1].lower() + title[1:]) + continue + parts += ["", f"### {title}", "", _changes_table(rows, years)] + parts += ["", "### Household net income by decile", "", _deciles(summary, years)] + if unchanged: + parts += ["", "**No change in any year:** " + "; ".join(unchanged) + "."] + + optional = [] + if pre_existing: + optional.append( + _details( + f"Checks failing on both base and head ({len(pre_existing)})", + "\n".join(_check_list(summary["checks"], "pre_existing")), + ) + ) + optional.append(_details("Counted-once check", _guard(summary))) + optional.append(_details("Provenance", _provenance(summary))) + for section, title in SECTIONS: + rows = _section_rows(summary, section) + if rows: + optional.append( + _details(f"{title}: head levels", _levels_table(rows, years)) + ) + + text = "\n".join(parts) + limit = spec.max_comment_characters + note = "\n\n_Some detail was left out to fit a GitHub comment; the workflow artifact has all of it._" + kept = [] + for block in optional: + candidate = text + "\n\n" + "\n\n".join(kept + [block]) + if len(candidate) + len(note) > limit: + continue + kept.append(block) + if kept: + text += "\n\n" + "\n\n".join(kept) + if len(kept) < len(optional): + text += note + if len(text) > limit: + text = text[: limit - len(note) - 1] + note + return text + "\n" diff --git a/policyengine_uk/utils/aggregate_impact/report_spec.yaml b/policyengine_uk/utils/aggregate_impact/report_spec.yaml new file mode 100644 index 0000000000..04fb81640a --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/report_spec.yaml @@ -0,0 +1,88 @@ +# What the aggregate-impact report measures and when it flags a move. +schema_version: 1 + +# Model periods are fiscal years: 2026 is 2026-27. The horizon spans the +# current fiscal year's predecessor (2025-26, the last year of outturn the +# Budget compares against) through 2031-32, the last year of the Autumn +# Budget 2026 scorecard. Every year is a full microsimulation run. +horizon: + first_year: 2025 + last_year: 2031 + +# Year of the counted-once check, which perturbs every component once. +guard_year: 2026 + +poverty: + measures: + - id: absolute_bhc + label: Absolute poverty, before housing costs + variable: in_poverty_bhc + - id: absolute_ahc + label: Absolute poverty, after housing costs + variable: in_poverty_ahc + - id: relative_bhc + label: Relative poverty, before housing costs + variable: in_relative_poverty_bhc + - id: relative_ahc + label: Relative poverty, after housing costs + variable: in_relative_poverty_ahc + groups: + - id: all + label: Everyone + - id: children + label: Children + variable: is_hbai_dependent_child + - id: working_age_adults + label: Working-age adults + variable: is_hbai_working_age_adult + - id: pensioners + label: Pensioners + variable: is_hbai_pensioner + +inequality: + - id: gini_equiv_hbai_bhc + label: Gini coefficient, equivalised HBAI income before housing costs + variable: equiv_hbai_household_net_income + +deciles: + # Households are placed in deciles by the base run's own decile variable + # (person-weighted equivalised HBAI income); head incomes are compared + # household by household within those base deciles. + decile_variable: household_income_decile + income_variable: household_net_income + +# A move is flagged for review when |head - base| exceeds +# max(absolute, relative * |base|). Flags never fail the job: intended policy +# changes move numbers. These thresholds are proposals awaiting the +# numerical-acceptance ruling (cos d1344). +flags: + gbp: + absolute: 100000000 + relative: 0.001 + count: + absolute: 5000 + relative: 0.005 + rate: + absolute: 0.001 + index: + absolute: 0.001 + decile_relative: + absolute: 0.001 + gbp_household: + absolute: 10 + relative: 0.001 + reform_gbp: + absolute: 100000000 + relative: 0.01 + +# Relative tolerance for accounting identities between weighted totals. The +# model stores values as float32, so totals summed in different orders differ +# by a few parts in ten million. +identity_relative_tolerance: 0.000001 +identity_absolute_tolerance_gbp: 1000000 + +# Tolerance on the counted-once coefficient (1 per unit perturbed). +counted_once_tolerance: 0.001 + +comment: + max_characters: 60000 diff --git a/policyengine_uk/utils/aggregate_impact/spec.py b/policyengine_uk/utils/aggregate_impact/spec.py new file mode 100644 index 0000000000..2441253702 --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/spec.py @@ -0,0 +1,105 @@ +"""The report specification: horizon, measures and flag thresholds.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping, Optional + +import yaml + +DEFAULT_SPEC_PATH = Path(__file__).with_name("report_spec.yaml") + + +class SpecError(ValueError): + pass + + +@dataclass(frozen=True) +class Threshold: + absolute: float + relative: float = 0.0 + + def limit(self, base: float) -> float: + return max(self.absolute, self.relative * abs(base)) + + def exceeded(self, base: float, head: float) -> bool: + return abs(head - base) > self.limit(base) + + +@dataclass(frozen=True) +class Spec: + raw: Mapping + first_year: int + last_year: int + guard_year: int + poverty_measures: tuple + poverty_groups: tuple + inequality: tuple + decile_variable: str + income_variable: str + flags: Mapping[str, Threshold] + identity_relative_tolerance: float + identity_absolute_tolerance: float + counted_once_tolerance: float + max_comment_characters: int + + @property + def years(self) -> list: + return list(range(self.first_year, self.last_year + 1)) + + def threshold(self, kind: str) -> Optional[Threshold]: + return self.flags.get(kind) + + +def fiscal_year_label(year: int) -> str: + return f"{year}-{(year + 1) % 100:02d}" + + +def parse_spec(raw: Mapping) -> Spec: + if raw.get("schema_version") != 1: + raise SpecError( + f"unsupported spec schema_version {raw.get('schema_version')!r}" + ) + horizon = raw.get("horizon") or {} + first, last = int(horizon["first_year"]), int(horizon["last_year"]) + if last < first: + raise SpecError("horizon.last_year is before horizon.first_year") + guard_year = int(raw.get("guard_year", first)) + poverty = raw.get("poverty") or {} + flags = { + kind: Threshold(float(v.get("absolute", 0.0)), float(v.get("relative", 0.0))) + for kind, v in (raw.get("flags") or {}).items() + } + deciles = raw.get("deciles") or {} + return Spec( + raw=raw, + first_year=first, + last_year=last, + guard_year=guard_year, + poverty_measures=tuple(poverty.get("measures") or ()), + poverty_groups=tuple(poverty.get("groups") or ()), + inequality=tuple(raw.get("inequality") or ()), + decile_variable=str(deciles.get("decile_variable", "household_income_decile")), + income_variable=str(deciles.get("income_variable", "household_net_income")), + flags=flags, + identity_relative_tolerance=float(raw.get("identity_relative_tolerance", 1e-6)), + identity_absolute_tolerance=float( + raw.get("identity_absolute_tolerance_gbp", 1e6) + ), + counted_once_tolerance=float(raw.get("counted_once_tolerance", 1e-3)), + max_comment_characters=int( + (raw.get("comment") or {}).get("max_characters", 60000) + ), + ) + + +def load_spec(path=None, years: Optional[str] = None) -> Spec: + """Load the spec, optionally overriding the horizon with ``"2025-2031"``.""" + with open(path or DEFAULT_SPEC_PATH) as f: + raw = yaml.safe_load(f) + if years: + start, _, end = years.partition("-") + raw = dict(raw) + raw["horizon"] = {"first_year": int(start), "last_year": int(end or start)} + return parse_spec(raw) diff --git a/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml b/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml new file mode 100644 index 0000000000..c2447136de --- /dev/null +++ b/policyengine_uk/utils/aggregate_impact/standard_reforms.yaml @@ -0,0 +1,106 @@ +# The standard reforms the aggregate-impact report costs on each side. +# +# Each reform is either absolute (`parameters`: values applied from 2023, as +# the reform-cost fixtures in tests/microsimulation/reforms_config.yaml do) or +# relative (`scale`: each listed parameter is multiplied by the factor in +# every year of the horizon, starting from that year's baseline value). +# +# `responds` states what the reform must do: the model variable whose +# weighted total it must move, the direction, and the least it must move it +# in every year of the horizon. A reform that moves its target by less, by +# nothing or the wrong way fails the `responsive:` check, so a reform the +# model silently ignores cannot pass as a small number. Each minimum is well +# below the reform's smallest yearly effect on uk-data 1.58.0 in October 2026 +# (about a fifth of it for most), and is a proposal awaiting the +# numerical-acceptance ruling (cos d1344). An absolute reform shrinks as the +# baseline is uprated towards it: the 13,000 personal allowance is worth £4.4bn +# in 2030-31 but £1.8bn in 2031-32, when the baseline allowance rises to +# 12,830. From 2032-33 the baseline (13,090) exceeds it, so before the horizon +# reaches that year, make that reform relative. +schema_version: 1 + +reforms: + - id: basic_rate_plus_1pp + name: Raise basic rate by 1pp + parameters: + gov.hmrc.income_tax.rates.uk[0].rate: 0.21 + responds: + variable: income_tax + direction: up + minimum: 1500000000 + - id: higher_rate_plus_1pp + name: Raise higher rate by 1pp + parameters: + gov.hmrc.income_tax.rates.uk[1].rate: 0.42 + responds: + variable: income_tax + direction: up + minimum: 1000000000 + - id: personal_allowance_13000 + name: Raise personal allowance by ~800GBP/year + parameters: + gov.hmrc.income_tax.allowances.personal_allowance.amount: 13000 + responds: + variable: income_tax + direction: down + minimum: 1000000000 + - id: child_benefit_additional_25 + name: Raise child benefit by 25GBP/week per additional child + parameters: + gov.hmrc.child_benefit.amount.additional: 25 + responds: + variable: child_benefit + direction: up + minimum: 300000000 + - id: uc_taper_20pc + name: Reduce Universal Credit taper rate to 20% + parameters: + gov.dwp.universal_credit.means_test.reduction_rate: 0.2 + responds: + variable: universal_credit + direction: up + minimum: 4000000000 + - id: employee_nics_main_10pc + name: Raise Class 1 main employee NICs rate to 10% + parameters: + gov.hmrc.national_insurance.class_1.rates.employee.main: 0.1 + responds: + variable: national_insurance + direction: up + minimum: 2500000000 + - id: vat_standard_plus_2pp + name: Raise VAT standard rate by 2pp + parameters: + gov.hmrc.vat.standard_rate: 0.22 + responds: + variable: vat + direction: up + minimum: 4000000000 + - id: additional_rate_plus_3pp + name: Raise additional rate by 3pp + parameters: + gov.hmrc.income_tax.rates.uk[2].rate: 0.48 + responds: + variable: income_tax + direction: up + minimum: 750000000 + - id: uc_standard_allowance_plus_10pc + name: Raise the Universal Credit standard allowance by 10% + scale: + gov.dwp.universal_credit.standard_allowance.amount.SINGLE_YOUNG: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.SINGLE_OLD: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.COUPLE_YOUNG: 1.1 + gov.dwp.universal_credit.standard_allowance.amount.COUPLE_OLD: 1.1 + responds: + variable: universal_credit + direction: up + minimum: 600000000 + - id: state_pension_plus_10pc + name: Raise the State Pension by 10% + scale: + gov.dwp.state_pension.new_state_pension.amount: 1.1 + gov.dwp.state_pension.basic_state_pension.amount: 1.1 + responds: + variable: state_pension + direction: up + minimum: 2500000000