From f6a21397371cb6d4fa0a3b81d71bb8d52332ff9c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 07:53:40 -0400 Subject: [PATCH 01/21] Limit salary sacrifice to the pay it comes out of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 1 of the salary sacrifice imputation ran the QRF for everyone not asked SALSAC and kept positive draws even without pay. The forest learnt the no-pay case from the 30 SALSAC respondents with a job but no recorded pay, three of whom report a sacrifice, so about 1.5k records (about 1.06m weighted, £11.3bn of the £25.8bn total in the 1.56.16 enhanced FRS) had a sacrifice and no pay. From 2029 policyengine-uk returns the excess over the £2,000 cap to employment income, so those people gained pay. - limit_salary_sacrifice_to_pay: every amount, reported or imputed, is between zero and the person's pay. No FRS 2024-25 respondent who reports a sacrifice and has pay reports more than that pay. - The QRF trains only on SALSAC respondents with pay. - Stage 2 moves employee pension to salary sacrifice up to the person's pay; any remainder stays as employee pension. - Tests: Hypothesis properties of the limit and of the whole imputation (stub model), and built-dataset checks that nobody has a sacrifice without pay or above it, and nobody without pay has employment income in 2029. Both built-dataset checks fail on the 1.56.16 file. - hypothesis added to the dev extras. Co-Authored-By: Claude Opus 5.5 --- .../salary-sacrifice-needs-pay.fixed.md | 1 + .../datasets/imputations/salary_sacrifice.py | 60 ++++- ...sion_contributions_via_salary_sacrifice.py | 30 +++ .../tests/test_salary_sacrifice_imputation.py | 209 ++++++++++++++++++ pyproject.toml | 1 + uv.lock | 73 +++++- 6 files changed, 361 insertions(+), 13 deletions(-) create mode 100644 changelog.d/salary-sacrifice-needs-pay.fixed.md create mode 100644 policyengine_uk_data/tests/test_salary_sacrifice_imputation.py diff --git a/changelog.d/salary-sacrifice-needs-pay.fixed.md b/changelog.d/salary-sacrifice-needs-pay.fixed.md new file mode 100644 index 000000000..58afe644b --- /dev/null +++ b/changelog.d/salary-sacrifice-needs-pay.fixed.md @@ -0,0 +1 @@ +- Limit every salary sacrifice amount, reported or imputed, to the person's pay, and train the salary sacrifice QRF only on SALSAC respondents with pay. Stage 1 had drawn amounts for people without pay from the three SALSAC respondents with a job but no recorded pay, so about a million people with no pay carried salary sacrifice, and from 2029 policyengine-uk returned the excess over the £2,000 cap to their employment income. diff --git a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py index 35800b339..578620eef 100644 --- a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py +++ b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py @@ -12,6 +12,14 @@ employee pension contributions to salary sacrifice. Targets the OBR/ASHE estimate of ~4.3mn below-cap SS users. +A salary sacrifice is pay given up for an employer pension contribution, +so every amount, reported or imputed, is limited to the person's pay: +nobody without pay sacrifices anything. Without that limit, stage 1 drew +amounts for people with no pay from the few SALSAC respondents who have a +job but no recorded pay, and from 2029 policyengine-uk returns the excess +over the £2,000 cap to employment income, giving those people pay they do +not have. + Exact monetary totals (~£24bn SS contributions) and final headcount calibration happen via weight optimisation in a subsequent step. """ @@ -33,14 +41,35 @@ ] +def limit_salary_sacrifice_to_pay( + salary_sacrifice: np.ndarray, employment_income: np.ndarray +) -> np.ndarray: + """Limit salary sacrifice amounts to the pay they are sacrificed from. + + A person without pay sacrifices nothing, and nobody sacrifices more + than the pay they still receive. No FRS 2024-25 respondent who reports + a sacrifice and has pay reports more than that pay, so the upper bound + only binds on imputed amounts. + + Args: + salary_sacrifice: Annual salary sacrifice pension contributions. + employment_income: Annual employment income after the sacrifice. + + Returns: + Amounts between zero and the person's pay (zero without pay). + """ + return np.clip(salary_sacrifice, 0, np.maximum(employment_income, 0)) + + def save_salary_sacrifice_model(): """ Train and save salary sacrifice imputation model using FRS data. - Uses FRS respondents who were asked about salary sacrifice (SALSAC field) - as training data. The model learns to predict the SS pension amount - directly - non-participants have 0, participants have their reported - SPNAMT value. + Uses FRS respondents with pay who were asked about salary sacrifice + (SALSAC field) as training data. The model learns to predict the SS + pension amount directly - non-participants have 0, participants have + their reported SPNAMT value. Respondents without pay are left out: the + model only predicts for people with pay. Returns: Trained QRF model for salary sacrifice imputation. @@ -71,9 +100,9 @@ def save_salary_sacrifice_model(): ) ss_asked = dataset.person.salary_sacrifice_asked.values - # Build training DataFrame with only those who were asked + # Build training DataFrame with only those who were asked and have pay # This includes both participants (with amounts) and non-participants (0) - training_mask = ss_asked == 1 + training_mask = (ss_asked == 1) & (employment_income > 0) if training_mask.sum() == 0: raise ValueError("No training data found - no respondents were asked SALSAC.") @@ -119,9 +148,12 @@ def impute_salary_sacrifice( Impute salary sacrifice pension amounts for FRS non-respondents. Stage 1: QRF predicts SS amounts for respondents not asked SALSAC. + Every amount, reported or predicted, is then limited to the person's + pay (see ``limit_salary_sacrifice_to_pay``). Stage 2: Converts a fraction of pension contributors to below-cap SS users, targeting ~4.3mn (OBR/ASHE). Moves employee pension - contributions to salary sacrifice to keep total pension consistent. + contributions (up to the person's pay) to salary sacrifice to keep + total pension consistent. Args: dataset: PolicyEngine UK dataset with salary_sacrifice_asked @@ -166,9 +198,13 @@ def impute_salary_sacrifice( # For those not asked, use the imputed values final_ss = np.where( ss_asked == 1, - current_ss, # Keep reported values exactly + current_ss, # Keep reported values imputed_ss, # Use imputed for non-respondents ) + # A sacrifice comes out of pay. This also covers SPI-synthetic rows + # copied from a SALSAC respondent, whose kept amounts come from the + # FRS-only imputation rather than the survey. + final_ss = limit_salary_sacrifice_to_pay(final_ss, employment_income) # Stage 2: Headcount-targeted imputation for SS users. # ASHE data shows many more SS users than the FRS captures due to @@ -198,15 +234,15 @@ def impute_salary_sacrifice( rng = np.random.default_rng(seed=2024) newly_imputed = is_donor & (rng.random(len(final_ss)) < imputation_rate) - # Move full employee pension to SS so the above/below - # 2k split reflects the natural pension distribution - ss_new = employee_pension.copy() + # Move full employee pension (up to pay) to SS so the + # above/below 2k split reflects the natural pension distribution + ss_new = limit_salary_sacrifice_to_pay(employee_pension, employment_income) final_ss = np.where(newly_imputed, ss_new, final_ss) # Reduce employee pension correspondingly dataset.person["employee_pension_contributions"] = np.where( newly_imputed, - 0.0, + employee_pension - ss_new, employee_pension, ) diff --git a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py index 9ed48abfe..5362beb69 100644 --- a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py +++ b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py @@ -20,3 +20,33 @@ def test_pension_contributions_via_salary_sacrifice(baseline): assert total < total_employment, ( f"Salary sacrifice contributions ({total / 1e9:.1f}B) cannot exceed total employment income ({total_employment / 1e9:.1f}B)" ) + + +def test_salary_sacrifice_needs_pay(baseline): + """A salary sacrifice comes out of pay: none without pay, none above it.""" + pay = baseline.calculate("employment_income_before_lsr", period=2025).values + ss = baseline.calculate( + "pension_contributions_via_salary_sacrifice", period=2025 + ).values + weight = baseline.calculate("person_weight", period=2025).values + + without_pay = (ss > 0) & (pay <= 0) + assert not without_pay.any(), ( + f"{without_pay.sum()} people ({weight[without_pay].sum() / 1e3:.0f}k " + "weighted) have salary sacrifice but no pay" + ) + # Both columns uprate by the same index, so only rounding can separate them. + above_pay = ss > pay * (1 + 1e-9) + 1e-6 + assert not above_pay.any(), ( + f"{above_pay.sum()} people sacrifice more than their pay" + ) + + +def test_salary_sacrifice_cap_gives_nobody_pay_they_do_not_have(baseline): + """From 2029 the excess over the cap returns to pay, so it needs pay.""" + pay = baseline.calculate("employment_income_before_lsr", period=2029).values + employment_income = baseline.calculate("employment_income", period=2029).values + phantom = (pay <= 0) & (employment_income > 0) + assert not phantom.any(), ( + f"{phantom.sum()} people without pay have employment income in 2029" + ) diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py new file mode 100644 index 000000000..d91ab8e73 --- /dev/null +++ b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py @@ -0,0 +1,209 @@ +"""Salary sacrifice imputation keeps every sacrifice within the person's pay. + +Invariants of ``limit_salary_sacrifice_to_pay``, for every input: + +- the result is between zero and the person's pay, so zero without pay; +- an amount already within those bounds is unchanged; +- applying it twice changes nothing; +- it is non-decreasing in the sacrifice and in the pay. + +``impute_salary_sacrifice`` applies it to every record, reported or +imputed, and stage 2 moves employee pension contributions to salary +sacrifice without changing anyone's total of the two. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.imputations import ( + salary_sacrifice as salary_sacrifice_module, +) +from policyengine_uk_data.datasets.imputations.salary_sacrifice import ( + limit_salary_sacrifice_to_pay, +) + +amounts = st.floats(min_value=-1e5, max_value=1e7, allow_nan=False) +pairs = st.lists(st.tuples(amounts, amounts), min_size=1, max_size=40) + + +def _split(rows): + ss, pay = np.array(rows, dtype=float).T + return ss, pay + + +@given(pairs) +def test_result_is_between_zero_and_pay(rows): + ss, pay = _split(rows) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.all(limited >= 0) + assert np.all(limited <= np.maximum(pay, 0)) + assert np.all(limited[pay <= 0] == 0) + + +@given(pairs) +def test_amounts_within_pay_are_unchanged(rows): + ss, pay = _split(rows) + within = (ss >= 0) & (ss <= pay) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.array_equal(limited[within], ss[within]) + + +@given(pairs) +def test_limit_is_idempotent(rows): + ss, pay = _split(rows) + once = limit_salary_sacrifice_to_pay(ss, pay) + assert np.array_equal(limit_salary_sacrifice_to_pay(once, pay), once) + + +@given(pairs, st.floats(min_value=0, max_value=1e6), st.floats(0, 1e6)) +def test_limit_is_monotone_in_sacrifice_and_pay(rows, more_ss, more_pay): + ss, pay = _split(rows) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.all(limit_salary_sacrifice_to_pay(ss + more_ss, pay) >= limited) + assert np.all(limit_salary_sacrifice_to_pay(ss, pay + more_pay) >= limited) + + +class _StubModel: + """Stands in for the trained QRF: predicts a fixed amount per row.""" + + def __init__(self, predictions): + self.predictions = np.asarray(predictions, dtype=float) + + def predict(self, X): + assert len(X) == len(self.predictions) + return pd.DataFrame( + {"pension_contributions_via_salary_sacrifice": self.predictions} + ) + + +def _dataset(pay, asked, reported, employee_pension): + from policyengine_uk.data import UKSingleYearDataset + + n = len(pay) + person = pd.DataFrame( + { + "person_id": np.arange(n), + "person_benunit_id": np.arange(n), + "person_household_id": np.arange(n), + "age": 40, + "employment_income": np.asarray(pay, dtype=float), + "salary_sacrifice_asked": np.asarray(asked, dtype=int), + "pension_contributions_via_salary_sacrifice": np.asarray( + reported, dtype=float + ), + "employee_pension_contributions": np.asarray(employee_pension, dtype=float), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household = pd.DataFrame( + { + "household_id": np.arange(n), + "household_weight": 1.0, + "region": "LONDON", + "tenure_type": "OWNED_OUTRIGHT", + "council_tax": 0.0, + "rent": 0.0, + } + ) + return UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2025 + ) + + +def _impute(monkeypatch, predictions, **columns): + monkeypatch.setattr( + salary_sacrifice_module, + "create_salary_sacrifice_model", + lambda: _StubModel(predictions), + ) + dataset = _dataset(**columns) + before = dataset.person.copy() + after = salary_sacrifice_module.impute_salary_sacrifice(dataset).person + return before, after + + +def _assert_invariants(before, after): + pay = before.employment_income.values + ss = after.pension_contributions_via_salary_sacrifice.values + assert np.all(ss >= 0) + assert np.all(ss <= np.maximum(pay, 0)) + assert np.all(ss[pay <= 0] == 0) + # Stage 2 moves employee pension to salary sacrifice: whoever lost + # employee pension gained exactly that much salary sacrifice. + pension_before = before.employee_pension_contributions.values + pension_after = after.employee_pension_contributions.values + moved = pension_after != pension_before + assert np.all(pension_after <= pension_before) + assert np.allclose(pension_after[moved] + ss[moved], pension_before[moved]) + + +def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch): + pay = [0, 0, -500, 30_000, 30_000, 8_000, 0] + before, after = _impute( + monkeypatch, + # The model predicts a large sacrifice for everyone. + predictions=[25_000] * 7, + pay=pay, + asked=[0, 1, 0, 0, 1, 0, 1], + reported=[0, 6_000, 0, 0, 1_200, 0, 0], + employee_pension=[0] * 7, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + # Unpaid: imputed (0, 2) and reported (1) amounts are all dropped. + assert ss[[0, 1, 2, 6]].tolist() == [0, 0, 0, 0] + # Paid and not asked: the prediction, limited to pay. + assert ss[3] == 25_000 + assert ss[5] == 8_000 + # Paid and asked: the reported amount is kept. + assert ss[4] == 1_200 + _assert_invariants(before, after) + + +def test_stage_two_moves_employee_pension_up_to_pay(monkeypatch): + n = 40 + # Every donor's employee pension exceeds their pay, so each one moved + # keeps the part above their pay as employee pension. + before, after = _impute( + monkeypatch, + predictions=[0] * n, + pay=[1_000] * n, + asked=[0] * n, + reported=[0] * n, + employee_pension=[1_500] * n, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + moved = ss > 0 + assert moved.any() + assert np.all(ss[moved] == 1_000) + assert np.all(after.employee_pension_contributions.values[moved] == 500) + _assert_invariants(before, after) + + +row = st.tuples( + st.sampled_from([-1_000.0, 0.0, 500.0, 15_000.0, 60_000.0]), # pay + st.integers(0, 1), # asked SALSAC + st.sampled_from([0.0, 800.0, 6_000.0, 70_000.0]), # reported amount + st.sampled_from([0.0, 300.0, 3_000.0, 90_000.0]), # employee pension + st.sampled_from([-100.0, 0.0, 1_500.0, 27_000.0]), # model prediction +) + + +@settings(max_examples=10, deadline=None) +@given(st.lists(row, min_size=1, max_size=12)) +def test_imputation_keeps_every_sacrifice_within_pay(rows): + pay, asked, reported, employee_pension, predictions = map(list, zip(*rows)) + with pytest.MonkeyPatch.context() as monkeypatch: + before, after = _impute( + monkeypatch, + predictions=predictions, + pay=pay, + asked=asked, + reported=reported, + employee_pension=employee_pension, + ) + _assert_invariants(before, after) diff --git a/pyproject.toml b/pyproject.toml index beff7f1a4..6880a9a25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dev = [ "itables", "quantile-forest", "build", "towncrier>=24.8.0", + "hypothesis", ] diff --git a/uv.lock b/uv.lock index a3e9f44d9..c301c6c9f 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f4/124858007ddf3c61e9b144107304c9152fa80b5b6c168da07d86fe583cc1/huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d", size = 516000, upload-time = "2025-11-20T15:49:30.926Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, + { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, + { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, + { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, + { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, + { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, + { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, + { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, + { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, + { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, + { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1364,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1392,6 +1461,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "furo" }, + { name = "hypothesis" }, { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, @@ -1410,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "microcalibrate", specifier = ">=0.18.0" }, From f95ef2c521c62a459705ba4366cabb8490dbaa0a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 08:27:04 -0400 Subject: [PATCH 02/21] Check the model's salary sacrifice pay limit never binds on the dataset policyengine-uk is gaining the same limit (pension_contributions_via_salary_sacrifice_from_pay). Where the installed model has it, the built dataset's salary sacrifice must already equal it. Skipped on a policyengine-uk without the variable. Co-Authored-By: Claude Opus 5.5 --- ...sion_contributions_via_salary_sacrifice.py | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py index 5362beb69..41d5c784a 100644 --- a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py +++ b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py @@ -1,3 +1,7 @@ +import numpy as np +import pytest + + def test_pension_contributions_via_salary_sacrifice(baseline): """Test that pension_contributions_via_salary_sacrifice loads and has reasonable values.""" values = baseline.calculate( @@ -50,3 +54,20 @@ def test_salary_sacrifice_cap_gives_nobody_pay_they_do_not_have(baseline): assert not phantom.any(), ( f"{phantom.sum()} people without pay have employment income in 2029" ) + + +def test_model_pay_limit_never_binds_on_the_dataset(baseline): + """policyengine-uk limits salary sacrifice to pay as the imputation does.""" + limited = "pension_contributions_via_salary_sacrifice_from_pay" + if limited not in baseline.tax_benefit_system.variables: + pytest.skip(f"policyengine-uk has no {limited} variable") + for period in (2025, 2029): + ss = baseline.calculate( + "pension_contributions_via_salary_sacrifice", period=period + ).values + from_pay = baseline.calculate(limited, period=period).values + differing = ~np.isclose(from_pay, ss, rtol=1e-6, atol=1e-2) + assert not differing.any(), ( + f"policyengine-uk limits the salary sacrifice of {differing.sum()} " + f"people in {period}" + ) From a99a1ef2b9c69fd72e368f5a3b613907455715fc Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 11:00:00 -0400 Subject: [PATCH 03/21] Take stage 2's salary sacrifice out of the donor's pay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 2 converts a pension contributor's employee contribution into salary sacrifice but left their pay unchanged. FRS pay is before that deduction, and policyengine-uk treats employment income as pay after any sacrifice, so a converted donor kept the same pay, lost the relief on the contribution and paid more income tax. Removing the phantom no-pay users enlarges stage 2's shortfall: at pre-calibration weights the extra donors (1.32m weighted) moved £2.95bn and paid £0.90bn more income tax for that reason alone. Stage 2 now takes the amount moved out of pay, so taxable pay is unchanged, and moves at most half the pay, so the sacrifice never exceeds the pay that remains. Tests: pay falls by exactly the amount moved, pay plus salary sacrifice and the pension total are unchanged for every donor, nobody else's pay changes. Co-Authored-By: Claude Opus 5.5 --- .../salary-sacrifice-needs-pay.fixed.md | 1 + .../datasets/imputations/salary_sacrifice.py | 29 ++++++++---- .../tests/test_salary_sacrifice_imputation.py | 47 +++++++++++++++---- 3 files changed, 59 insertions(+), 18 deletions(-) diff --git a/changelog.d/salary-sacrifice-needs-pay.fixed.md b/changelog.d/salary-sacrifice-needs-pay.fixed.md index 58afe644b..f60a7a5ad 100644 --- a/changelog.d/salary-sacrifice-needs-pay.fixed.md +++ b/changelog.d/salary-sacrifice-needs-pay.fixed.md @@ -1 +1,2 @@ - Limit every salary sacrifice amount, reported or imputed, to the person's pay, and train the salary sacrifice QRF only on SALSAC respondents with pay. Stage 1 had drawn amounts for people without pay from the three SALSAC respondents with a job but no recorded pay, so about a million people with no pay carried salary sacrifice, and from 2029 policyengine-uk returned the excess over the £2,000 cap to their employment income. +- Stage 2 of the salary sacrifice imputation now takes the employee pension contribution it converts to salary sacrifice out of the donor's pay (at most half the pay). It had left pay unchanged, so converted donors lost the income tax relief on that contribution. diff --git a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py index 578620eef..0e10a1097 100644 --- a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py +++ b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py @@ -9,8 +9,9 @@ 2. Headcount-targeted imputation: converts a fraction of pension contributors without SS into below-cap (≤£2,000) SS users, moving - employee pension contributions to salary sacrifice. Targets the - OBR/ASHE estimate of ~4.3mn below-cap SS users. + employee pension contributions to salary sacrifice and taking the + amount moved out of their pay. Targets the OBR/ASHE estimate of + ~4.3mn below-cap SS users. A salary sacrifice is pay given up for an employer pension contribution, so every amount, reported or imputed, is limited to the person's pay: @@ -152,8 +153,8 @@ def impute_salary_sacrifice( pay (see ``limit_salary_sacrifice_to_pay``). Stage 2: Converts a fraction of pension contributors to below-cap SS users, targeting ~4.3mn (OBR/ASHE). Moves employee pension - contributions (up to the person's pay) to salary sacrifice to keep - total pension consistent. + contributions to salary sacrifice to keep total pension consistent, + and takes the amount moved out of pay (at most half the pay). Args: dataset: PolicyEngine UK dataset with salary_sacrifice_asked @@ -234,17 +235,29 @@ def impute_salary_sacrifice( rng = np.random.default_rng(seed=2024) newly_imputed = is_donor & (rng.random(len(final_ss)) < imputation_rate) - # Move full employee pension (up to pay) to SS so the - # above/below 2k split reflects the natural pension distribution - ss_new = limit_salary_sacrifice_to_pay(employee_pension, employment_income) + # Move full employee pension to SS so the above/below 2k + # split reflects the natural pension distribution. The FRS + # pay these contributions were deducted from is before the + # sacrifice, so the sacrifice comes out of it: pay falls by + # the amount moved, which keeps taxable pay unchanged. At most + # half the pay moves, so the sacrifice never exceeds the pay + # that remains. + ss_new = limit_salary_sacrifice_to_pay( + employee_pension, employment_income / 2 + ) final_ss = np.where(newly_imputed, ss_new, final_ss) - # Reduce employee pension correspondingly + # Reduce employee pension and pay correspondingly dataset.person["employee_pension_contributions"] = np.where( newly_imputed, employee_pension - ss_new, employee_pension, ) + dataset.person["employment_income"] = np.where( + newly_imputed, + dataset.person["employment_income"].values - ss_new, + dataset.person["employment_income"].values, + ) dataset.person["pension_contributions_via_salary_sacrifice"] = final_ss diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py index d91ab8e73..f71a5bdb5 100644 --- a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py +++ b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py @@ -8,8 +8,10 @@ - it is non-decreasing in the sacrifice and in the pay. ``impute_salary_sacrifice`` applies it to every record, reported or -imputed, and stage 2 moves employee pension contributions to salary -sacrifice without changing anyone's total of the two. +imputed. Stage 2 moves employee pension contributions to salary sacrifice +and takes the amount moved out of pay, so for every donor the pension total +and pay plus salary sacrifice are both unchanged, and the sacrifice never +exceeds the pay that remains. """ from __future__ import annotations @@ -128,18 +130,22 @@ def _impute(monkeypatch, predictions, **columns): def _assert_invariants(before, after): - pay = before.employment_income.values + pay_before = before.employment_income.values + pay = after.employment_income.values ss = after.pension_contributions_via_salary_sacrifice.values assert np.all(ss >= 0) assert np.all(ss <= np.maximum(pay, 0)) assert np.all(ss[pay <= 0] == 0) - # Stage 2 moves employee pension to salary sacrifice: whoever lost - # employee pension gained exactly that much salary sacrifice. + # Stage 2 moves employee pension to salary sacrifice out of pay: whoever + # lost employee pension gained exactly that much salary sacrifice and + # lost exactly that much pay. Nobody else's pay changes. pension_before = before.employee_pension_contributions.values pension_after = after.employee_pension_contributions.values moved = pension_after != pension_before assert np.all(pension_after <= pension_before) assert np.allclose(pension_after[moved] + ss[moved], pension_before[moved]) + assert np.allclose(pay[moved] + ss[moved], pay_before[moved]) + assert np.array_equal(pay[~moved], pay_before[~moved]) def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch): @@ -164,10 +170,30 @@ def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch): _assert_invariants(before, after) -def test_stage_two_moves_employee_pension_up_to_pay(monkeypatch): +def test_stage_two_takes_the_sacrifice_out_of_pay(monkeypatch): n = 40 - # Every donor's employee pension exceeds their pay, so each one moved - # keeps the part above their pay as employee pension. + before, after = _impute( + monkeypatch, + predictions=[0] * n, + pay=[30_000] * n, + asked=[0] * n, + reported=[0] * n, + employee_pension=[1_500] * n, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + moved = ss > 0 + assert moved.any() and not moved.all() + assert np.all(ss[moved] == 1_500) + assert np.all(after.employee_pension_contributions.values[moved] == 0) + assert np.all(after.employment_income.values[moved] == 28_500) + assert np.all(after.employment_income.values[~moved] == 30_000) + _assert_invariants(before, after) + + +def test_stage_two_moves_at_most_half_the_pay(monkeypatch): + n = 40 + # Every donor's employee pension exceeds half their pay, so each one + # moved sacrifices half and keeps the rest as employee pension. before, after = _impute( monkeypatch, predictions=[0] * n, @@ -179,8 +205,9 @@ def test_stage_two_moves_employee_pension_up_to_pay(monkeypatch): ss = after.pension_contributions_via_salary_sacrifice.values moved = ss > 0 assert moved.any() - assert np.all(ss[moved] == 1_000) - assert np.all(after.employee_pension_contributions.values[moved] == 500) + assert np.all(ss[moved] == 500) + assert np.all(after.employee_pension_contributions.values[moved] == 1_000) + assert np.all(after.employment_income.values[moved] == 500) _assert_invariants(before, after) From 9910ad42b414591a3c89ade0c17e9fa5d4431faa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 09:00:55 -0400 Subject: [PATCH 04/21] Limit salary sacrifice against stored pay, not the simulation's pay Review (subfleet 20261002-093645-ukdata-518-ss-review) found the limits used employment_income from a Microsimulation of the 2024 survey-year dataset, which calculates 2025 by default and uprates pay by 5.3% but not salary sacrifice. Sacrifices could therefore reach 105.3% of stored pay, and stage 2 could move up to 52.65% of stored pay and leave less than that. CI on a99a1ef caught it: 4 people in the TESTING build sacrificed more than their pay. - The limits and stage-2 donor pay use the stored employment_income column. QRF predictors still come from the simulation, as in training. - The model cache is now salary_sacrifice_paid_respondents.pkl, so a forest cached from the old training set is not reused. - The no-training-data error names the actual condition; docstrings say the upper bound is an imputation bound, not a legal one, and that only predictions for people with pay are kept. - Wiring tests run at 2024 and 2025; the 2024 cases fail on a99a1ef. Co-Authored-By: Claude Opus 5.5 --- .../datasets/imputations/salary_sacrifice.py | 45 +++++++++++-------- .../tests/test_salary_sacrifice_imputation.py | 26 ++++++++--- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py index 0e10a1097..bc74d6719 100644 --- a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py +++ b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py @@ -32,6 +32,10 @@ from policyengine_uk.data import UKSingleYearDataset from policyengine_uk import Microsimulation +# Trained only on SALSAC respondents with pay; the name changed with that +# training set so a forest cached by an older build is not reused. +MODEL_FILE = "salary_sacrifice_paid_respondents.pkl" + PREDICTORS = [ "age", "employment_income", @@ -47,10 +51,12 @@ def limit_salary_sacrifice_to_pay( ) -> np.ndarray: """Limit salary sacrifice amounts to the pay they are sacrificed from. - A person without pay sacrifices nothing, and nobody sacrifices more - than the pay they still receive. No FRS 2024-25 respondent who reports - a sacrifice and has pay reports more than that pay, so the upper bound - only binds on imputed amounts. + A person without pay sacrifices nothing. The upper bound is an + imputation bound, not a legal one (an employee may sacrifice, for + example, a whole bonus, as long as cash pay stays above the National + Minimum Wage): no FRS 2024-25 respondent who reports a sacrifice and + has pay reports more than the pay they still receive, so imputed + amounts are held to the same bound. Args: salary_sacrifice: Annual salary sacrifice pension contributions. @@ -69,8 +75,8 @@ def save_salary_sacrifice_model(): Uses FRS respondents with pay who were asked about salary sacrifice (SALSAC field) as training data. The model learns to predict the SS pension amount directly - non-participants have 0, participants have - their reported SPNAMT value. Respondents without pay are left out: the - model only predicts for people with pay. + their reported SPNAMT value. Respondents without pay are left out: only + predictions for people with pay are kept. Returns: Trained QRF model for salary sacrifice imputation. @@ -106,7 +112,10 @@ def save_salary_sacrifice_model(): training_mask = (ss_asked == 1) & (employment_income > 0) if training_mask.sum() == 0: - raise ValueError("No training data found - no respondents were asked SALSAC.") + raise ValueError( + "No training data found - no SALSAC respondents with positive " + "employment income." + ) train_df = pd.DataFrame( { @@ -119,7 +128,7 @@ def save_salary_sacrifice_model(): # Train QRF model model = QRF() model.fit(train_df[PREDICTORS], train_df[IMPUTATIONS]) - model.save(STORAGE_FOLDER / "salary_sacrifice.pkl") + model.save(STORAGE_FOLDER / MODEL_FILE) return model @@ -136,7 +145,7 @@ def create_salary_sacrifice_model(overwrite_existing: bool = False): """ from policyengine_uk_data.utils.qrf import QRF - model_path = STORAGE_FOLDER / "salary_sacrifice.pkl" + model_path = STORAGE_FOLDER / MODEL_FILE if model_path.exists() and not overwrite_existing: return QRF(file_path=model_path) return save_salary_sacrifice_model() @@ -166,9 +175,13 @@ def impute_salary_sacrifice( dataset = dataset.copy() sim = Microsimulation(dataset=dataset) - # Get variables needed for imputation + # Get variables needed for imputation. The predictors come from the + # simulation, as in training; the limits use the stored pay, which is in + # the same year as the stored salary sacrifice (the simulation may + # calculate a later year and uprate pay but not salary sacrifice). age = sim.calculate("age").values employment_income = sim.calculate("employment_income").values + pay = dataset.person["employment_income"].values current_ss = dataset.person.pension_contributions_via_salary_sacrifice.values # Get indicator for who was asked @@ -205,7 +218,7 @@ def impute_salary_sacrifice( # A sacrifice comes out of pay. This also covers SPI-synthetic rows # copied from a SALSAC respondent, whose kept amounts come from the # FRS-only imputation rather than the survey. - final_ss = limit_salary_sacrifice_to_pay(final_ss, employment_income) + final_ss = limit_salary_sacrifice_to_pay(final_ss, pay) # Stage 2: Headcount-targeted imputation for SS users. # ASHE data shows many more SS users than the FRS captures due to @@ -219,7 +232,7 @@ def impute_salary_sacrifice( has_ss = final_ss > 0 # Donor pool: employed pension contributors not already SS users - is_donor = (employee_pension > 0) & ~has_ss & (employment_income > 0) + is_donor = (employee_pension > 0) & ~has_ss & (pay > 0) # Create enough SS records for the calibrator to work with. # Target ~70% of the 7.7mn total so the calibrator can gently @@ -242,9 +255,7 @@ def impute_salary_sacrifice( # the amount moved, which keeps taxable pay unchanged. At most # half the pay moves, so the sacrifice never exceeds the pay # that remains. - ss_new = limit_salary_sacrifice_to_pay( - employee_pension, employment_income / 2 - ) + ss_new = limit_salary_sacrifice_to_pay(employee_pension, pay / 2) final_ss = np.where(newly_imputed, ss_new, final_ss) # Reduce employee pension and pay correspondingly @@ -254,9 +265,7 @@ def impute_salary_sacrifice( employee_pension, ) dataset.person["employment_income"] = np.where( - newly_imputed, - dataset.person["employment_income"].values - ss_new, - dataset.person["employment_income"].values, + newly_imputed, pay - ss_new, pay ) dataset.person["pension_contributions_via_salary_sacrifice"] = final_ss diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py index f71a5bdb5..574b1fb8e 100644 --- a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py +++ b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py @@ -70,6 +70,12 @@ def test_limit_is_monotone_in_sacrifice_and_pay(rows, more_ss, more_pay): assert np.all(limit_salary_sacrifice_to_pay(ss, pay + more_pay) >= limited) +# The build imputes on the survey-year dataset (2024) while the simulation +# calculates a later default year, uprating pay but not salary sacrifice, so +# the wiring is checked in both years. +YEARS = pytest.mark.parametrize("year", [2024, 2025]) + + class _StubModel: """Stands in for the trained QRF: predicts a fixed amount per row.""" @@ -83,7 +89,7 @@ def predict(self, X): ) -def _dataset(pay, asked, reported, employee_pension): +def _dataset(pay, asked, reported, employee_pension, year): from policyengine_uk.data import UKSingleYearDataset n = len(pay) @@ -113,17 +119,17 @@ def _dataset(pay, asked, reported, employee_pension): } ) return UKSingleYearDataset( - person=person, benunit=benunit, household=household, fiscal_year=2025 + person=person, benunit=benunit, household=household, fiscal_year=year ) -def _impute(monkeypatch, predictions, **columns): +def _impute(monkeypatch, predictions, year=2024, **columns): monkeypatch.setattr( salary_sacrifice_module, "create_salary_sacrifice_model", lambda: _StubModel(predictions), ) - dataset = _dataset(**columns) + dataset = _dataset(year=year, **columns) before = dataset.person.copy() after = salary_sacrifice_module.impute_salary_sacrifice(dataset).person return before, after @@ -148,10 +154,12 @@ def _assert_invariants(before, after): assert np.array_equal(pay[~moved], pay_before[~moved]) -def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch): +@YEARS +def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch, year): pay = [0, 0, -500, 30_000, 30_000, 8_000, 0] before, after = _impute( monkeypatch, + year=year, # The model predicts a large sacrifice for everyone. predictions=[25_000] * 7, pay=pay, @@ -170,10 +178,12 @@ def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch): _assert_invariants(before, after) -def test_stage_two_takes_the_sacrifice_out_of_pay(monkeypatch): +@YEARS +def test_stage_two_takes_the_sacrifice_out_of_pay(monkeypatch, year): n = 40 before, after = _impute( monkeypatch, + year=year, predictions=[0] * n, pay=[30_000] * n, asked=[0] * n, @@ -190,12 +200,14 @@ def test_stage_two_takes_the_sacrifice_out_of_pay(monkeypatch): _assert_invariants(before, after) -def test_stage_two_moves_at_most_half_the_pay(monkeypatch): +@YEARS +def test_stage_two_moves_at_most_half_the_pay(monkeypatch, year): n = 40 # Every donor's employee pension exceeds half their pay, so each one # moved sacrifices half and keeps the rest as employee pension. before, after = _impute( monkeypatch, + year=year, predictions=[0] * n, pay=[1_000] * n, asked=[0] * n, From 321e613f2ef3b3175db997d90199800f39eb5d84 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 14:33:08 -0400 Subject: [PATCH 05/21] Correct the stage 2 comments after the pay change Review r2 (subfleet 20261003-090151-ukdata-518-ss-review-r2, APPROVE) nits: the stage 2 comments still said donors keep their full employee pension and become below-cap users, and "keeps taxable pay unchanged" claimed more than the code guarantees (pay plus sacrifice is unchanged). The stored pay array is copied before use. No change in behaviour. Co-Authored-By: Claude Opus 5.5 --- .../datasets/imputations/salary_sacrifice.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py index bc74d6719..83a57b925 100644 --- a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py +++ b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py @@ -8,10 +8,10 @@ not asked. 2. Headcount-targeted imputation: converts a fraction of pension - contributors without SS into below-cap (≤£2,000) SS users, moving - employee pension contributions to salary sacrifice and taking the - amount moved out of their pay. Targets the OBR/ASHE estimate of - ~4.3mn below-cap SS users. + contributors without SS into SS users, moving employee pension + contributions to salary sacrifice and taking the amount moved out of + their pay, so the calibrator has enough records to reach the OBR + headcounts (7.7mn users: 4.3mn at or below £2,000, 3.3mn above). A salary sacrifice is pay given up for an employer pension contribution, so every amount, reported or imputed, is limited to the person's pay: @@ -160,8 +160,8 @@ def impute_salary_sacrifice( Stage 1: QRF predicts SS amounts for respondents not asked SALSAC. Every amount, reported or predicted, is then limited to the person's pay (see ``limit_salary_sacrifice_to_pay``). - Stage 2: Converts a fraction of pension contributors to below-cap - SS users, targeting ~4.3mn (OBR/ASHE). Moves employee pension + Stage 2: Converts a fraction of pension contributors to SS users, + for the OBR headcount targets. Moves employee pension contributions to salary sacrifice to keep total pension consistent, and takes the amount moved out of pay (at most half the pay). @@ -181,7 +181,7 @@ def impute_salary_sacrifice( # calculate a later year and uprate pay but not salary sacrifice). age = sim.calculate("age").values employment_income = sim.calculate("employment_income").values - pay = dataset.person["employment_income"].values + pay = dataset.person["employment_income"].values.copy() current_ss = dataset.person.pension_contributions_via_salary_sacrifice.values # Get indicator for who was asked @@ -225,8 +225,9 @@ def impute_salary_sacrifice( # self-reporting bias in auto-enrolment. Impute additional SS users # from pension contributors to create enough records for calibration # to hit OBR headcount targets (7.7mn total, 3.3mn above 2k, - # 4.3mn below 2k). Donors keep their full employee pension amount - # so those above 2k become above-cap records and the rest below-cap. + # 4.3mn below 2k). Donors move their employee pension amount (at + # most half their pay), so those above 2k become above-cap records + # and the rest below-cap. person_weight = sim.calculate("person_weight").values employee_pension = dataset.person["employee_pension_contributions"].values.copy() has_ss = final_ss > 0 @@ -252,7 +253,7 @@ def impute_salary_sacrifice( # split reflects the natural pension distribution. The FRS # pay these contributions were deducted from is before the # sacrifice, so the sacrifice comes out of it: pay falls by - # the amount moved, which keeps taxable pay unchanged. At most + # the amount moved, which keeps pay plus sacrifice unchanged. At most # half the pay moves, so the sacrifice never exceeds the pay # that remains. ss_new = limit_salary_sacrifice_to_pay(employee_pension, pay / 2) From 14e2c22b2795e28f5d4bc2cc4b827cfdaf511bf3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 14:41:04 -0400 Subject: [PATCH 06/21] Describe the source of the no-pay draws without a respondent count Co-Authored-By: Claude Opus 5.5 --- changelog.d/salary-sacrifice-needs-pay.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/salary-sacrifice-needs-pay.fixed.md b/changelog.d/salary-sacrifice-needs-pay.fixed.md index f60a7a5ad..7f9682293 100644 --- a/changelog.d/salary-sacrifice-needs-pay.fixed.md +++ b/changelog.d/salary-sacrifice-needs-pay.fixed.md @@ -1,2 +1,2 @@ -- Limit every salary sacrifice amount, reported or imputed, to the person's pay, and train the salary sacrifice QRF only on SALSAC respondents with pay. Stage 1 had drawn amounts for people without pay from the three SALSAC respondents with a job but no recorded pay, so about a million people with no pay carried salary sacrifice, and from 2029 policyengine-uk returned the excess over the £2,000 cap to their employment income. +- Limit every salary sacrifice amount, reported or imputed, to the person's pay, and train the salary sacrifice QRF only on SALSAC respondents with pay. Stage 1 had drawn amounts for people without pay from the few SALSAC respondents who report a sacrifice but have no recorded pay, so about a million people with no pay carried salary sacrifice, and from 2029 policyengine-uk returned the excess over the £2,000 cap to their employment income. - Stage 2 of the salary sacrifice imputation now takes the employee pension contribution it converts to salary sacrifice out of the donor's pay (at most half the pay). It had left pay unchanged, so converted donors lost the income tax relief on that contribution. From 9fc0e299e9ba1bfe30cc85c80933b56768acddc1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 16:55:28 -0400 Subject: [PATCH 07/21] Take SPI Scottish taxpayer status from SCOT_TXP; keep unknown regions simulable create_spi labels GORCODE 13 (address abroad), 14 (address unknown) and -1 (composite records) UNKNOWN. policyengine-uk 2.104.5 (PolicyEngine/policyengine-uk#1985) uprates their rent by the UK-wide index, so the dataset now simulates as built. Scottish taxpayer status comes from HMRC's SCOT_TXP flag rather than the region: the two disagree for 1,394 GORCODE-11 records and 606 Scottish taxpayers elsewhere on the 2022-23 tape. load_spi_dataset relabels UNKNOWN as SOUTH_EAST only on releases before 2.104.5, and ensure_spi_dataset rebuilds a cached H5 without the flag. Co-Authored-By: Claude Opus 5.5 --- changelog.d/spi-unknown-region.fixed.md | 1 + policyengine_uk_data/datasets/spi.py | 39 +++- .../tests/test_spi_allowance_deductions.py | 1 + policyengine_uk_data/tests/test_spi_build.py | 175 +++++++++++++++++- .../utils/incomes_projection.py | 18 +- 5 files changed, 217 insertions(+), 17 deletions(-) create mode 100644 changelog.d/spi-unknown-region.fixed.md diff --git a/changelog.d/spi-unknown-region.fixed.md b/changelog.d/spi-unknown-region.fixed.md new file mode 100644 index 000000000..a8a4b44cf --- /dev/null +++ b/changelog.d/spi-unknown-region.fixed.md @@ -0,0 +1 @@ +Take Scottish taxpayer status in the SPI dataset from HMRC's `SCOT_TXP` flag rather than the region code. `GORCODE` 13, 14 and -1 (address abroad, address unknown, composite records) stay `UNKNOWN`, which policyengine-uk 2.104.5 and later can simulate; `load_spi_dataset` relabels them `SOUTH_EAST` only for older releases, and rebuilds a cached SPI H5 that predates the flag. diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index e22be675c..a21ddb14f 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -1,3 +1,6 @@ +import re +from importlib.metadata import version + from policyengine_uk_data.storage import STORAGE_FOLDER import pandas as pd import numpy as np @@ -23,10 +26,11 @@ 7: (74, 90), } -# SPI GORCODE → policyengine-uk region enum. -# NB the SPI codebook does not include a "region unknown" code; we surface -# unknown codes explicitly rather than silently mapping them to SOUTH_EAST -# (which the previous implementation did, distorting regional income totals). +# SPI GORCODE → policyengine-uk region enum. GORCODE also takes 13 ("Address +# abroad"), 14 ("Address unknown or not available") and -1 (composite +# records). None of those is a UK region, so they become "UNKNOWN" rather +# than SOUTH_EAST (which the previous implementation used, distorting +# regional income totals). REGION_MAP = { 1: "NORTH_EAST", 2: "NORTH_WEST", @@ -42,6 +46,16 @@ 12: "NORTHERN_IRELAND", } +# First policyengine-uk release that can simulate Region.UNKNOWN: it uprates +# their rent by the UK-wide index (PolicyEngine/policyengine-uk#1985). +UNKNOWN_REGION_MODEL_VERSION = (2, 104, 5) + + +def model_simulates_unknown_region() -> bool: + """Whether the installed policyengine-uk can simulate Region.UNKNOWN.""" + release = re.match(r"\d+(\.\d+)*", version("policyengine-uk")).group() + return tuple(map(int, release.split("."))) >= UNKNOWN_REGION_MODEL_VERSION + def _get_marriage_allowance(fiscal_year: int) -> float: """Return the maximum Marriage Allowance transfer for the given UK fiscal @@ -98,10 +112,10 @@ def create_spi( existing call sites don't break. seed: Seed for the random age imputation. Fixed by default so builds are deterministic. - unknown_region: Fallback region label for SPI GORCODE values outside - the documented 1-12 range. Defaults to ``"UNKNOWN"`` so regional - totals are not silently distorted; pass ``"SOUTH_EAST"`` to - reproduce legacy behaviour if needed. + unknown_region: Region label for SPI GORCODE values outside 1-12 + (address abroad, address unknown, composite records). Defaults to + ``"UNKNOWN"`` so regional totals are not silently distorted; pass + ``"SOUTH_EAST"`` to reproduce legacy behaviour if needed. """ df = pd.read_csv(spi_data_file_path, delimiter="\t") rng = np.random.default_rng(seed) @@ -119,6 +133,15 @@ def create_spi( person["dividend_income"] = df.DIVIDENDS person["gift_aid"] = df.GIFTAID household["region"] = df.GORCODE.map(REGION_MAP).fillna(unknown_region) + # GORCODE is the address at the end of the tax year; SCOT_TXP marks + # records HMRC taxed under the Scottish system for the year. The two + # disagree for some records and SCOT_TXP is also set on records with no + # UK region, so SCOT_TXP decides the rates. HMRC documents it as "." or 1; + # the 2022-23 tape holds 0 or 1. WELSH_TXP is not used: policyengine-uk + # has no separate Welsh rates. + person["pays_scottish_income_tax"] = ( + pd.to_numeric(df.SCOT_TXP, errors="coerce") == 1 + ) household["rent"] = 0 household["tenure_type"] = "OWNED_OUTRIGHT" household["council_tax"] = 0 diff --git a/policyengine_uk_data/tests/test_spi_allowance_deductions.py b/policyengine_uk_data/tests/test_spi_allowance_deductions.py index f635bb1f6..f40892905 100644 --- a/policyengine_uk_data/tests/test_spi_allowance_deductions.py +++ b/policyengine_uk_data/tests/test_spi_allowance_deductions.py @@ -10,6 +10,7 @@ def test_spi_overrides_allowance_deductions_not_policy_parameters(tmp_path): "DIVIDENDS": 0, "GIFTAID": 0, "GORCODE": 7, + "SCOT_TXP": 0, "INCBBS": 0, "INCPROP": 1_000, "PAY": 0, diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index efb37f810..14c4a5c41 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -23,6 +23,7 @@ import numpy as np import pandas as pd import pytest +from policyengine_core.errors import ParameterNotFoundError if importlib.util.find_spec("policyengine_uk") is None: pytest.skip( @@ -30,6 +31,8 @@ allow_module_level=True, ) +from policyengine_uk_data.datasets.spi import model_simulates_unknown_region + SPI_COLUMNS = [ "SEX", @@ -38,6 +41,7 @@ "DIVIDENDS", "GIFTAID", "GORCODE", + "SCOT_TXP", "INCBBS", "INCPROP", "PAY", @@ -356,7 +360,17 @@ def save(self, path): assert dataset_path.read_text() == "rebuilt h5" -def test_income_projection_loads_local_h5_dataset(monkeypatch): +@pytest.mark.parametrize( + "model_simulates, expected", + [ + (False, ["SOUTH_EAST", "LONDON", "SOUTH_EAST"]), + (True, ["UNKNOWN", "LONDON", "SOUTH_EAST"]), + ], +) +def test_income_projection_loads_local_h5_dataset( + monkeypatch, model_simulates, expected +): + """UNKNOWN becomes SOUTH_EAST only for a model that cannot simulate it.""" from policyengine_uk_data.utils import incomes_projection calls = {} @@ -374,16 +388,55 @@ def __init__(self, path): lambda: "/tmp/spi_2022_23.h5", ) monkeypatch.setattr(incomes_projection, "UKSingleYearDataset", FakeDataset) + monkeypatch.setattr( + incomes_projection, + "model_simulates_unknown_region", + lambda: model_simulates, + ) dataset = incomes_projection.load_spi_dataset() assert isinstance(dataset, FakeDataset) assert calls == {"path": "/tmp/spi_2022_23.h5"} - assert dataset.household["region"].tolist() == [ - "SOUTH_EAST", - "LONDON", - "SOUTH_EAST", - ] + assert dataset.household["region"].tolist() == expected + + +def test_income_projection_rebuilds_spi_dataset_without_scottish_flag( + tmp_path, monkeypatch +): + """A cached H5 from before create_spi read SCOT_TXP is rebuilt; a current + one is reused.""" + from policyengine_uk_data.datasets.spi import create_spi + from policyengine_uk_data.utils import incomes_projection + + tab_dir = tmp_path / "spi_2022_23" + tab_dir.mkdir() + tab = tab_dir / "put2223uk.tab" + _write_fake_spi(tab, gor_values=(11, 13), maind_values=(0, 0)) + dataset_path = tmp_path / "spi_2022_23.h5" + stale = create_spi(tab, 2022) + stale.person = stale.person.drop(columns="pays_scottish_income_tax") + stale.save(dataset_path) + + builds = [] + + def counting_create_spi(path, fiscal_year): + builds.append(fiscal_year) + return create_spi(path, fiscal_year) + + monkeypatch.setattr(incomes_projection, "STORAGE_FOLDER", tmp_path) + monkeypatch.setattr(incomes_projection, "SPI_RELEASE_NAME", "spi_2022_23") + monkeypatch.setattr(incomes_projection, "SPI_TAB_FILENAME", "put2223uk.tab") + monkeypatch.setattr(incomes_projection, "SPI_H5_FILENAME", "spi_2022_23.h5") + monkeypatch.setattr(incomes_projection, "SPI_FISCAL_YEAR", 2022) + monkeypatch.setattr(incomes_projection, "create_spi", counting_create_spi) + + assert not incomes_projection._has_scottish_taxpayer_flag(dataset_path) + assert incomes_projection.ensure_spi_dataset() == str(dataset_path) + assert builds == [2022] + assert incomes_projection._has_scottish_taxpayer_flag(dataset_path) + assert incomes_projection.ensure_spi_dataset() == str(dataset_path) + assert builds == [2022] def test_income_model_cache_rejects_stale_spi_release(tmp_path, monkeypatch): @@ -467,3 +520,113 @@ def test_income_model_cache_accepts_current_spi_release(tmp_path, monkeypatch): ) assert income_module.create_income_model().metadata == current_metadata + + +def _set_spi_columns(path, **columns): + df = pd.read_csv(path, sep="\t") + for col, values in columns.items(): + df[col] = list(values) + df.to_csv(path, sep="\t", index=False) + + +@pytest.mark.parametrize("not_scottish", [0, ".", ""]) +def test_create_spi_region_and_scottish_taxpayer_invariants(tmp_path, not_scottish): + """For every GORCODE (documented 1-14, composite -1, undocumented 99) and + SCOT_TXP value, the region follows GORCODE alone and is always a Region + member, and Scottish taxpayer status follows SCOT_TXP alone. HMRC + documents "not a Scottish taxpayer" as "."; the 2022-23 tape writes 0. + """ + from itertools import product + + from policyengine_uk.variables.household.demographic.geography import Region + + from policyengine_uk_data.datasets.spi import REGION_MAP, create_spi + + cases = list(product([-1, *range(1, 15), 99], (not_scottish, 1))) + gor = [g for g, _ in cases] + scot = [s for _, s in cases] + tab = tmp_path / "spi.tab" + _write_fake_spi(tab, gor_values=gor, maind_values=[0] * len(cases)) + _set_spi_columns(tab, SCOT_TXP=scot) + + ds = create_spi(tab, 2022) + + regions = ds.household["region"].tolist() + assert regions == [REGION_MAP.get(g, "UNKNOWN") for g in gor] + assert set(regions) <= {region.name for region in Region} + assert ds.person["pays_scottish_income_tax"].tolist() == [s == 1 for s in scot] + # Address abroad, address unknown and composite records stay UNKNOWN even + # when they are Scottish taxpayers. + assert {r for r, g in zip(regions, gor) if g in (-1, 13, 14)} == {"UNKNOWN"} + + +def test_create_spi_scottish_taxpayer_status_survives_h5_round_trip(tmp_path): + from policyengine_uk.data import UKSingleYearDataset + + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + _write_fake_spi(tab, gor_values=(13, 11, 7), maind_values=(0, 0, 0)) + _set_spi_columns(tab, SCOT_TXP=(1, 0, 1)) + ds = create_spi(tab, 2022) + ds.save(tmp_path / "spi.h5") + + loaded = UKSingleYearDataset(str(tmp_path / "spi.h5")) + + assert loaded.person["pays_scottish_income_tax"].tolist() == [True, False, True] + assert loaded.household["region"].tolist() == ["UNKNOWN", "SCOTLAND", "LONDON"] + + +# GORCODE, SCOT_TXP: abroad, unknown, composite, London, Scotland, abroad and +# Scottish, Scotland but not Scottish. +SIMULATED_RECORDS = ((13, 0), (14, 0), (-1, 0), (7, 0), (11, 1), (13, 1), (11, 0)) + + +def _spi_income_tax(tmp_path, years, **kwargs): + from policyengine_uk import Microsimulation + + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + gor, scot = zip(*SIMULATED_RECORDS) + _write_fake_spi(tab, gor_values=gor, maind_values=[0] * len(gor)) + _set_spi_columns( + tab, SCOT_TXP=scot, PAY=[60_000] * len(gor), AGERANGE=[3] * len(gor) + ) + sim = Microsimulation(dataset=create_spi(tab, 2022, **kwargs)) + return {year: sim.calculate("income_tax", year).values for year in years} + + +@pytest.mark.parametrize("year", [2022, 2026]) +def test_spi_income_tax_follows_scottish_taxpayer_flag(tmp_path, year): + """Equal pay, so income tax depends only on SCOT_TXP, in the data year + and after uprating. Uses the legacy SOUTH_EAST label so it runs on any + policyengine-uk release.""" + tax = _spi_income_tax(tmp_path, [year], unknown_region="SOUTH_EAST")[year] + abroad, unknown, composite, london, scotland, abroad_scot, scotland_ruk = tax + + assert london > 0 + assert abroad == unknown == composite == scotland_ruk == london + assert abroad_scot == scotland != london + + +@pytest.mark.xfail( + condition=not model_simulates_unknown_region(), + reason="policyengine-uk before 2.104.5 (PolicyEngine/policyengine-uk#1985) " + "has no rent index for Region.UNKNOWN", + raises=ParameterNotFoundError, + strict=True, +) +def test_create_spi_output_with_unknown_region_can_be_simulated(tmp_path): + """SPI records with an address abroad (13), an unknown address (14) or a + composite record (-1) keep region UNKNOWN and still run through + policyengine-uk, giving record for record the income tax of the legacy + SOUTH_EAST relabelling: the region label does not move income tax. + """ + years = [2022, 2026] + tax = _spi_income_tax(tmp_path, years) + legacy = _spi_income_tax(tmp_path, years, unknown_region="SOUTH_EAST") + + for year in years: + assert (tax[year] > 0).all() + assert (tax[year] == legacy[year]).all() diff --git a/policyengine_uk_data/utils/incomes_projection.py b/policyengine_uk_data/utils/incomes_projection.py index 144e3f833..7e0fe5e8b 100644 --- a/policyengine_uk_data/utils/incomes_projection.py +++ b/policyengine_uk_data/utils/incomes_projection.py @@ -7,6 +7,7 @@ SPI_RELEASE_NAME, SPI_TAB_FILENAME, create_spi, + model_simulates_unknown_region, ) from policyengine_uk_data.storage import STORAGE_FOLDER from policyengine_uk_data.targets.sources.hmrc_spi import ( @@ -31,6 +32,12 @@ def _read_spi_dataset_year(dataset_path) -> int: return int(store["time_period"].iloc[0]) +def _has_scottish_taxpayer_flag(dataset_path) -> bool: + # SPI H5s built before create_spi read SCOT_TXP lack this column. + with pd.HDFStore(dataset_path, mode="r") as store: + return "pays_scottish_income_tax" in store.select("person", stop=0) + + def ensure_spi_dataset() -> str: """Create the SPI H5 projection input from the current TAB release if needed. @@ -42,6 +49,7 @@ def ensure_spi_dataset() -> str: if ( dataset_path.exists() and _read_spi_dataset_year(dataset_path) == SPI_FISCAL_YEAR + and _has_scottish_taxpayer_flag(dataset_path) ): return str(dataset_path) @@ -64,9 +72,13 @@ def ensure_spi_dataset() -> str: def load_spi_dataset() -> UKSingleYearDataset: dataset = UKSingleYearDataset(ensure_spi_dataset()) - dataset.household["region"] = dataset.household["region"].replace( - {"UNKNOWN": "SOUTH_EAST"} - ) + # Older policyengine-uk releases cannot simulate an unknown region. The + # stand-in label does not move income tax, which follows the Scottish + # taxpayer flag. + if not model_simulates_unknown_region(): + dataset.household["region"] = dataset.household["region"].replace( + {"UNKNOWN": "SOUTH_EAST"} + ) return dataset From 8d8b608e8aed057605808f9b73118329ef831d83 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 17:16:38 -0400 Subject: [PATCH 08/21] Regenerate the uprating factor table from policyengine-uk's load-time uprating storage/uprating_factors.csv was last built on 2026-05-20 from each variable's `uprating` attribute. policyengine-uk 2.93.0 projects a saved dataset with the year-on-year growth parameters in data/uprating_indices.yaml instead, so the build calibrated 2025 values the model does not run on: earnings were uprated 3.7% for 2024-25 against the engine's 5.3%, self-employment income 4.7% against 0.7%, and GDP-indexed incomes and wealth 2.8% against 4.4%. Regenerating with the old generator would not fix it: on 2.93.0 it emits no employment_income row (an `adds` formula with no attribute) and indexes savings interest by household interest income where the engine uses GDP per head. The generator now compounds the growth parameter each variable is listed under in uprating_indices.yaml, keeps the household-weight and road-fuel overrides, and stores six decimal places. The table loses four rows the engine does not uprate at load (benunit_rent, housing_service_charges, pension_contributions_via_salary_sacrifice, water_and_sewerage_charges) and gains bus_fare_spending and private_pension_wealth. incomes_projection.csv is regenerated: the table is its only input that changed (rebuilding it from main's table reproduced main's file exactly). compute_ss_headcount deflated contributions through the salary sacrifice row, which no longer exists; the loss matrix would have skipped all three headcount targets. It now classifies contributions as simulated in the calibration year, the values test_salary_sacrifice_headcount checks. Co-Authored-By: Claude Opus 5.5 --- .../uprating-factors-load-time.fixed.md | 1 + .../datasets/create_datasets.py | 14 +- .../storage/incomes_projection.csv | 140 +++++------ .../storage/uprating_factors.csv | 154 ++++++------ .../storage/uprating_growth_factors.csv | 156 ++++++------ .../targets/compute/income.py | 24 +- .../tests/test_income_projection.py | 9 + .../tests/test_road_fuel_volume_uprating.py | 9 +- .../tests/test_salary_sacrifice_headcount.py | 41 ++++ .../tests/test_uprating_factors_table.py | 225 ++++++++++++++++++ policyengine_uk_data/utils/uprating.py | 97 ++++++-- 11 files changed, 596 insertions(+), 274 deletions(-) create mode 100644 changelog.d/uprating-factors-load-time.fixed.md create mode 100644 policyengine_uk_data/tests/test_uprating_factors_table.py diff --git a/changelog.d/uprating-factors-load-time.fixed.md b/changelog.d/uprating-factors-load-time.fixed.md new file mode 100644 index 000000000..83d7d1872 --- /dev/null +++ b/changelog.d/uprating-factors-load-time.fixed.md @@ -0,0 +1 @@ +Regenerate `storage/uprating_factors.csv` (last built 2026-05-20) from the uprating policyengine-uk applies to a dataset at load (the year-on-year growth in its `uprating_indices.yaml`) rather than from each variable's `uprating` attribute, keeping the household-weight and road-fuel overrides and storing six decimal places. The stale table moved the FRS to the 2025 calibration year with 2024-25 growth of 3.7% for earnings (policyengine-uk 2.93.0: 5.3%), 4.7% for self-employment income (0.7%) and 2.8% for GDP-indexed incomes and wealth (4.4%), and uprated salary-sacrificed pension contributions, water charges and service charges, which policyengine-uk carries unchanged, so calibration now fits the values the model runs on. The SPI income projections are regenerated with the new table, and the salary sacrifice headcount targets compare contributions with the £2,000 cap as simulated instead of deflating them through a table row that no longer exists. diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py index 29a73028b..c02b0bfe5 100644 --- a/policyengine_uk_data/datasets/create_datasets.py +++ b/policyengine_uk_data/datasets/create_datasets.py @@ -315,13 +315,15 @@ def main(): # materialisation on purpose. Rail and bus (subsidy and fares) are # fitted by simulating the saved base-year file at the calibration # year, the configuration consumers actually run: policyengine-uk - # re-uprates rail_usage (gov.dft.rail.ridership_index) and - # bus_fare_spending (CPI) at load, so a factor fitted on a - # calibration-year file and then down-rated would be uprated a - # second time. rail_usage, rail_subsidy_spending, - # bus_subsidy_spending and bus_fare_spending are not in + # re-uprates bus_fare_spending (CPI) at load, so a factor fitted + # on a calibration-year file and then down-rated would be uprated + # a second time. (rail_usage declares gov.dft.rail.ridership_index, + # but policyengine-uk carries it forward unchanged at load because + # uprating_indices.yaml does not list it.) rail_usage, + # rail_subsidy_spending and bus_subsidy_spending are not in # uprating_factors.csv, so `uprate_dataset` leaves them untouched - # in either direction: uprating the saved file back to the + # in either direction, and bus_fare_spending moves by the same CPI + # factor both ways: uprating the saved file back to the # calibration year reproduces the calibrated weights and monetary # levels but keeps these post-calibration scalings, which the # weight solve never saw. The saved file is therefore not diff --git a/policyengine_uk_data/storage/incomes_projection.csv b/policyengine_uk_data/storage/incomes_projection.csv index b8db818ea..aa9bda2d5 100644 --- a/policyengine_uk_data/storage/incomes_projection.csv +++ b/policyengine_uk_data/storage/incomes_projection.csv @@ -13,73 +13,73 @@ total_income_lower_bound,total_income_upper_bound,employment_income_count,employ 500000,1000000.0,44000,23100000000,15000,6670000000,3000,43000000,4000,640000000,13000,474000000,38000,653000000,32000,3820000000,2024 1000000,inf,21000,35500000000,11000,19500000000,2000,20000000,2000,370000000,7000,439000000,21000,1240000000,19000,11100000000,2024 12570,inf,27263000,1084200000000,3670000,107480000000,7977000,85220000000,9175000,127760000000,2351000,29358000000,19573000,18343000000,4092000,71770000000,2024 -12570,15000.0,1375891,17529896907,449188,5033426819,1153320,12181568310,1082502,4556072351,102180,688579832,1264606,281598319,116344,179852941,2025 -15000,20000.0,3136222,50411419508,658607,8402997371,2073953,22917865804,2165005,14244272179,234711,1736865546,2873184,1407991597,376347,878710084,2025 -20000,30000.0,7557283,178410785091,972229,16429272568,2347108,25705173808,2691081,31630663221,436036,3925932773,5311344,3216798319,852850,3422344538,2025 -30000,40000.0,5432746,176336241079,601952,13185276074,1173554,12697736459,1466943,26079586563,368253,3905378151,3733116,2600159664,674794,4830336134,2025 -40000,50000.0,3540896,144180808882,363195,9135512708,574637,6214664511,788102,17700602929,328797,3884823529,2397692,1921857143,634326,8632941176,2025 -50000,70000.0,3470078,180485329104,314634,8601822962,446153,4810687146,650513,18224289406,401639,5549747899,2256056,2404890756,650513,12846638655,2025 -70000,100000.0,1659163,122398096749,144671,5608974584,178056,1951115602,262026,9918621878,228641,3874546218,1082502,1664924370,339926,9701781513,2025 -100000,150000.0,800242,85367486122,85993,5211323401,74865,874388844,110274,5687235142,138601,2620714286,494714,1294941176,224594,7060512605,2025 -150000,200000.0,267085,40661062649,40467,3819544259,21245,273569119,29339,1895745047,54631,1181890756,147706,668025210,93075,3545672269,2025 -200000,300000.0,182103,37964155432,34397,4593917616,14164,184788197,20234,1644375538,41479,1079117647,112297,753326891,75876,3802605042,2025 -300000,500000.0,94087,30080888184,21245,5064820333,8093,99104285,10117,1173057709,23269,786214286,68795,691663025,49573,3525117647,2025 -500000,1000000.0,44514,23960983347,15175,6979824715,3035,44390461,4047,670318691,13152,487144538,38444,671108403,32374,3925932773,2025 -1000000,inf,21245,36823156225,11129,20405784400,2023,20646726,2023,387527993,7082,451173950,21245,1274386555,19222,11407815126,2025 -12570,inf,27581555,1124610309278,3712882,112472497809,8070207,87975699272,9282205,133812368648,2378470,30172129412,19801701,18851671429,4139813,73760260504,2025 -12570,15000.0,1385161,17918556701,452214,5189404032,1161091,12410509297,1089796,4724677003,102869,708285714,1273126,289657143,117128,185000000,2026 -15000,20000.0,3157352,51529103886,663044,8663391762,2087926,23348585287,2179591,14771403962,236292,1786571429,2892541,1448285714,378882,903857143,2026 -20000,30000.0,7608199,182366375892,978779,16938387379,2362921,26188278092,2709211,32801205857,438974,4038285714,5347128,3308857143,858596,3520285714,2026 -30000,40000.0,5469348,180245836638,606008,13593865031,1181461,12936378335,1476826,27044702842,370734,4017142857,3758267,2674571429,679340,4968571429,2026 -40000,50000.0,3564752,147377478192,365642,9418606486,578508,6331463217,793412,18355641688,331013,3996000000,2413846,1976857143,638600,8880000000,2026 -50000,70000.0,3493457,184486915147,316754,8868378615,449159,4901099434,654896,18898708010,404345,5708571429,2271256,2473714286,654896,13214285714,2026 -70000,100000.0,1670341,125111816019,145646,5782787029,179256,1987784964,263792,10285676141,230181,3985428571,1089796,1712571429,342216,9979428571,2026 -100000,150000.0,805634,87260190325,86573,5372813322,75369,890822150,111017,5897700258,139535,2695714286,498047,1332000000,226107,7262571429,2026 -150000,200000.0,268884,41562569389,40740,3937905346,21389,278710590,29537,1965900086,54999,1215714286,148701,687142857,93702,3647142857,2026 -200000,300000.0,183330,38805868358,34629,4736275197,14259,188261116,20370,1705228252,41759,1110000000,113054,774885714,76388,3911428571,2026 -300000,500000.0,94721,30747819191,21389,5221770377,8148,100966855,10185,1216468562,23426,808714286,69258,711457143,49907,3626000000,2026 -500000,1000000.0,44814,24492228390,15278,7196117441,3056,45224737,4074,695124892,13241,501085714,38703,690314286,32592,4038285714,2026 -1000000,inf,21389,37639571768,11204,21038124452,2037,21034762,2037,401869078,7130,464085714,21389,1310857143,19352,11734285714,2026 -12570,inf,27767379,1149544329897,3737897,115957826468,8124578,89629118836,9344742,138764306632,2394495,31035600000,19935110,19391171429,4167704,75871142857,2026 -12570,15000.0,1395755,18293814433,455673,5353812445,1169971,12658528698,1098130,4825839793,103655,730243697,1282863,298636975,118023,190735294,2027 -15000,20000.0,3181500,52608247423,668115,8937861525,2103895,23815198060,2196261,15087683032,238099,1841957983,2914664,1493184874,381780,931878151,2027 -20000,30000.0,7666388,186185567010,986265,17475021911,2380993,26711641067,2729932,33503531438,442331,4163478992,5388023,3411436975,865163,3629420168,2027 -30000,40000.0,5511178,184020618557,610643,14024539877,1190497,13194907033,1488121,27623772610,373570,4141680672,3787011,2757487395,684536,5122605042,2027 -40000,50000.0,3592016,150463917526,368438,9717002629,582933,6457995150,799480,18748664944,333544,4119882353,2432308,2038142857,643484,9155294118,2027 -50000,70000.0,3520175,188350515464,319176,9149342682,452594,4999046079,659905,19303359173,407437,5885546218,2288627,2550403361,659905,13623949580,2027 -70000,100000.0,1683116,127731958763,146759,5965994741,180627,2027510105,265809,10505908699,231942,4108983193,1098130,1765663866,344833,10288806723,2027 -100000,150000.0,811796,89087628866,87235,5543032428,75945,908624899,111866,6023979328,140602,2779285714,501856,1373294118,227836,7487722689,2027 -150000,200000.0,270941,42432989691,41052,4062664330,21552,284280517,29762,2007993109,55420,1253403361,149838,708445378,94419,3760210084,2027 -200000,300000.0,184732,39618556701,34894,4886327783,14368,192023444,20526,1741739879,42078,1144411765,113918,798908403,76972,4032689076,2027 -300000,500000.0,95445,31391752577,21552,5387204207,8210,102984640,10263,1242515073,23605,833785714,69788,733513445,50288,3738411765,2027 -500000,1000000.0,45157,25005154639,15394,7424101665,3079,46128537,4105,710008613,13342,516620168,38999,711715126,32841,4163478992,2027 -1000000,inf,21552,38427835052,11289,21704645048,2053,21455133,2053,410473730,7184,478473109,21552,1351495798,19500,12098067227,2027 -12570,inf,27979749,1173618556701,3766485,119631551271,8186717,91420323363,9416212,141735469423,2412808,31997752941,20087577,19992328571,4199579,78223268908,2027 -12570,15000.0,1401052,18709278351,457402,5543514461,1174411,12906548100,1102298,4927002584,104049,754453782,1287731,308537815,118471,197058824,2028 -15000,20000.0,3193574,53803013481,670650,9254557406,2111879,24281810833,2204596,15403962102,239003,1903025210,2925725,1542689076,383229,962773109,2028 -20000,30000.0,7695482,190413957177,990008,18094215600,2390029,27235004042,2740292,34205857020,444010,4301512605,5408471,3524537815,868446,3749747899,2028 -30000,40000.0,5532093,188199841396,612960,14521472393,1195015,13453435732,1493768,28202842377,374987,4278991597,3801383,2848907563,687133,5292436975,2028 -40000,50000.0,3605648,153881046788,369836,10061305872,585145,6584527082,802514,19141688200,334810,4256470588,2441538,2105714286,645926,9458823529,2028 -50000,70000.0,3533535,192628072958,320388,9473531989,454312,5096992724,662409,19708010336,408983,6080672269,2297313,2634957983,662409,14075630252,2028 -70000,100000.0,1689503,130632831086,147316,6177388256,181313,2067235247,266818,10726141258,232822,4245210084,1102298,1824201681,346142,10629915966,2028 -100000,150000.0,814876,91110864393,87566,5739439089,76234,926427648,112290,6150258398,141135,2871428571,503760,1418823529,228701,7735966387,2028 -150000,200000.0,271969,43396669310,41207,4206617003,21634,289850445,29875,2050086133,55630,1294957983,150407,731932773,94777,3884873950,2028 -200000,300000.0,185433,40518318795,35026,5059465381,14423,195785772,20604,1778251507,42238,1182352941,114351,825394958,77264,4166386555,2028 -300000,500000.0,95807,32104678826,21634,5578089395,8241,105002425,10302,1268561585,23694,861428571,70053,757831933,50479,3862352941,2028 -500000,1000000.0,45328,25573037272,15453,7687160386,3091,47032336,4121,724892334,13392,533747899,39147,735310924,32966,4301512605,2028 -1000000,inf,21634,39300555115,11332,22473707274,2060,21875505,2060,419078381,7211,494336134,21634,1396302521,19574,12499159664,2028 -12570,inf,28085934,1200272164948,3780779,123870464505,8217786,93211527890,9451947,144706632214,2421965,33058588235,20163811,20655142857,4215517,80816638655,2028 -12570,15000.0,1408997,19178350515,459996,5754294479,1181071,13164106710,1108549,5043152455,104639,779226891,1295034,318668908,119143,203529412,2029 -15000,20000.0,3211685,55151942902,674454,9606441718,2123856,24766370251,2217098,15767097330,240358,1965512605,2942317,1593344538,385402,994386555,2029 -20000,30000.0,7739124,195187946075,995622,18782208589,2403583,27778496362,2755833,35012230835,446528,4442756303,5439143,3640268908,873371,3872873950,2029 -30000,40000.0,5563466,192918318795,616436,15073619632,1201792,13721907842,1502240,28867700258,377114,4419495798,3822941,2942453782,691030,5466218487,2029 -40000,50000.0,3626095,157739095956,371934,10443865031,588463,6715925627,807065,19592937123,336709,4396235294,2455385,2174857143,649589,9769411765,2029 -50000,70000.0,3553574,197457573354,322204,9833742331,456888,5198706548,666166,20172609819,411303,6280336134,2310341,2721478992,666166,14537815126,2029 -70000,100000.0,1699085,133908009516,148152,6412269939,182341,2108488278,268331,10979000861,234142,4384605042,1108549,1884100840,348105,10978957983,2029 -100000,150000.0,819498,93395162569,88062,5957668712,76666,944915117,112927,6295245478,141936,2965714286,506617,1465411765,229998,7989983193,2029 -150000,200000.0,273511,44484694687,41441,4366564417,21757,295634600,30045,2098415159,55945,1337478992,151260,755966387,95315,4012436975,2029 -200000,300000.0,186485,41534179223,35225,5251840491,14504,199692805,20721,1820172265,42477,1221176471,114999,852497479,77702,4303193277,2029 -300000,500000.0,96351,32909595559,21757,5790184049,8288,107097817,10360,1298466839,23829,889714286,70450,782715966,50765,3989176471,2029 -500000,1000000.0,45585,26214195083,15540,7979447853,3108,47970897,4144,741981051,13468,551273950,39369,759455462,33153,4442756303,2029 -1000000,inf,21757,40285884219,11396,23328220859,2072,22312045,2072,428957795,7252,510568067,21757,1442151261,19685,12909579832,2029 -12570,inf,28245211,1230364948454,3802220,128580368098,8264389,95071624899,9505550,148117967270,2435700,34144094118,20278162,21333371429,4239424,83470319328,2029 +12570,15000.0,1375891,17795688094,449188,4843188790,1153320,12201198978,1082502,4506163433,102180,699345737,1264606,286001092,116344,182664931,2025 +15000,20000.0,3136222,51175765762,658607,8085406650,2073953,22954798078,2165005,14088235102,234711,1764021335,2873184,1430005461,376347,892448664,2025 +20000,30000.0,7557283,181115878830,972229,15808329316,2347108,25746597844,2691081,31284169123,436036,3987314498,5311344,3267092769,852850,3475852690,2025 +30000,40000.0,5432746,179009880239,601952,12686939451,1173554,12718198935,1466943,25793901032,368253,3966438506,3733116,2640813005,674794,4905858152,2025 +40000,50000.0,3540896,146366902077,363195,8790236620,574637,6224679479,788102,17506703913,328797,3945562513,2397692,1951905265,634326,8767916697,2025 +50000,70000.0,3470078,183221877421,314634,8276717642,446153,4818439597,650513,18024653733,401639,5636517876,2256056,2442491080,650513,13047495084,2025 +70000,100000.0,1659163,124253916871,144671,5396983767,178056,1954259836,262026,9809969589,228641,3935124517,1082502,1690955363,339926,9853468288,2025 +100000,150000.0,800242,86661842021,85993,5014361783,74865,875797927,110274,5624935044,138601,2661688997,494714,1315187504,224594,7170903298,2025 +150000,200000.0,267085,41277572384,40467,3675184841,21245,274009977,29339,1874978348,54631,1200369548,147706,678469744,93075,3601108643,2025 +200000,300000.0,182103,38539774216,34397,4420290809,14164,185085985,20234,1626362435,41479,1095989587,112297,765105112,75876,3862058545,2025 +300000,500000.0,94087,30536979570,21245,4873395789,8093,99263992,10117,1160207597,23269,798506699,68795,702477135,49573,3580232651,2025 +500000,1000000.0,44514,24324283727,15175,6716022709,3035,44461996,4047,662975769,13152,494761014,38444,681601143,32374,3987314498,2025 +1000000,inf,21245,37381474991,11129,19634549151,2023,20679998,2023,383282867,7082,458228027,21245,1294311512,19222,11586175635,2025 +12570,inf,27581555,1141661836204,3712882,108221607319,8070207,88117472623,9282205,132346537984,2378470,30643868854,19801701,19146416186,4139813,74913497775,2025 +12570,15000.0,1385161,18400750621,452214,4991391904,1161091,12481831123,1089796,4690916055,102869,719766739,1273126,294352368,117128,187998775,2026 +15000,20000.0,3157352,52915768057,663044,8332822659,2087926,23482767029,2179591,14665852495,236292,1815531028,2892541,1471761839,378882,918508301,2026 +20000,30000.0,7608199,187273911644,978779,16292069209,2362921,26338779235,2709211,32566819510,438974,4103744689,5347128,3362492376,858596,3577348119,2026 +30000,40000.0,5469348,185096308020,606008,13075163824,1181461,13010722273,1476826,26851450524,370734,4082259115,3758267,2717925148,679340,5049109958,2026 +40000,50000.0,3564752,151343451852,365642,9059220649,578508,6367849438,793412,18224478468,331013,4060773541,2413846,2008901196,638600,9023941202,2026 +50000,70000.0,3493457,189451515267,316754,8529987828,449159,4929265511,654896,18763664221,404345,5801105058,2271256,2513812192,654896,13428483931,2026 +70000,100000.0,1670341,128478613802,145646,5562133182,179256,1999208544,263792,10212178171,230181,4050030754,1089796,1740331518,342216,10141191065,2026 +100000,150000.0,805634,89608389118,86573,5167802845,75369,895941607,111017,5855557283,139535,2739410722,498047,1353591180,226107,7380294769,2026 +150000,200000.0,268884,42681031026,40740,3787646663,21389,280312309,29537,1951852428,54999,1235420522,148701,698281164,93702,3706261565,2026 +200000,300000.0,183330,39850146315,34629,4555553110,14259,189343031,20370,1693043266,41759,1127992650,113054,787446298,76388,3974831244,2026 +300000,500000.0,94721,31575252545,21389,5022523247,8148,101547101,10185,1207776088,23426,821823217,69258,722989575,49907,3684775991,2026 +500000,1000000.0,44814,25151321854,15278,6921535135,3056,45484639,4074,690157764,13241,509208111,38703,701504001,32592,4103744689,2026 +1000000,inf,21389,38652464322,11204,20235372584,2037,21155646,2037,398997458,7130,471608356,21389,1332105606,19352,11924493731,2026 +12570,inf,27767379,1180478924442,3737897,111533222839,8124578,90144207486,9344742,137772743731,2394495,31538674501,19935110,19705494460,4167704,77100983340,2026 +12570,15000.0,1395755,18842365727,455673,5176573740,1169971,12731467935,1098130,4836334659,103655,743015304,1282863,303859990,118023,194071161,2027 +15000,20000.0,3181500,54185738125,668115,8641972377,2103895,23952422725,2196261,15120494566,238099,1874172930,2914664,1519299949,381780,948176246,2027 +20000,30000.0,7666388,191768455916,986265,16896508881,2380993,26865555219,2729932,33576392346,442331,4236296209,5388023,3471101344,865163,3692896957,2027 +30000,40000.0,5511178,189538590149,610643,13560255535,1190497,13270936915,1488121,27683846669,373570,4214116647,3787011,2805714505,684536,5212196906,2027 +40000,50000.0,3592016,154975670769,368438,9395319907,582933,6495206523,799480,18789438101,333544,4191937086,2432308,2073788982,643484,9315415747,2027 +50000,70000.0,3520175,193998321682,319176,8846452421,452594,5027850896,659905,19345338636,407437,5988481552,2288627,2595008672,659905,13862225814,2027 +70000,100000.0,1683116,131562080221,146759,5768489656,180627,2039192746,265809,10528756143,231942,4180847305,1098130,1796544465,344833,10468752935,2027 +100000,150000.0,811796,91758976290,87235,5359529569,75945,913860453,111866,6037079816,140602,2827894066,501856,1397312362,227836,7618679307,2027 +150000,200000.0,270941,43705369023,41052,3928169262,21552,285918560,29762,2012359939,55420,1275324775,149838,720835742,94419,3825974325,2027 +200000,300000.0,184732,40806543526,34894,4724565222,14368,193129895,20526,1745527682,42078,1164426968,113918,812880922,76972,4103218841,2027 +300000,500000.0,95445,32333053614,21552,5208860063,8210,103578044,10263,1245217200,23605,848368220,69788,746342238,50288,3803794763,2027 +500000,1000000.0,45157,25754949603,15394,7178325748,3079,46394332,4105,711552685,13342,525655603,38999,724162677,32841,4236296209,2027 +1000000,inf,21552,39580117355,11289,20986109757,2053,21578759,2053,411366396,7184,486841371,21552,1375132801,19500,12309656523,2027 +12570,inf,27979749,1208810232000,3766485,115671132138,8186717,91947093002,9416212,142043704838,2412808,32557378036,20087577,20341984648,4199579,79591355733,2027 +12570,15000.0,1401052,19238055016,457402,5364999173,1174411,12986098807,1102298,4981427321,104049,766048722,1287731,313279627,118471,200087353,2028 +15000,20000.0,3193574,55323637503,670650,8956537080,2111879,24431474027,2204596,15574117602,239003,1932272150,2925725,1566398134,383229,977569638,2028 +20000,30000.0,7695482,195795589515,990008,17511535761,2390029,27402869517,2740292,34583702323,444010,4367621074,5408471,3578705225,868446,3807376486,2028 +30000,40000.0,5532093,193518896614,612960,14053843987,1195015,13536357231,1493768,28514377081,374987,4344753948,3801383,2892691444,687133,5373774619,2028 +40000,50000.0,3605648,158230156643,369836,9737306191,585145,6625111425,802514,19353131432,334810,4321886822,2441538,2138076285,645926,9604192937,2028 +50000,70000.0,3533535,198072282416,320388,9168460125,454312,5128408512,662409,19925709286,408983,6174124031,2297313,2675453747,662409,14291953775,2028 +70000,100000.0,1689503,134324881179,147316,5978460617,181313,2079976843,266818,10844624536,232822,4310453259,1102298,1852237209,346142,10793283491,2028 +100000,150000.0,814876,93685912890,87566,5554614528,76234,932137770,112290,6218195484,141135,2915558570,503760,1440628941,228701,7854857795,2028 +150000,200000.0,271969,44623180866,41207,4071153218,21634,291636965,29875,2072731828,55630,1314859747,150407,743181596,94777,3944579242,2028 +200000,300000.0,185433,41663480094,35026,4896537706,14423,196992516,20604,1797894459,42238,1200524117,114351,838080169,77264,4230418317,2028 +300000,500000.0,95807,33012047069,21634,5398460706,8241,105649617,10302,1282574391,23694,874667571,70053,769478791,50479,3921712116,2028 +500000,1000000.0,45328,26295803010,15453,7439614237,3091,47322224,4121,732899652,13392,541950887,39147,746611665,32966,4367621074,2028 +1000000,inf,21634,40411298999,11332,21749996646,2060,22010337,2060,423707611,7211,501933417,21634,1417761814,19574,12691254952,2028 +12570,inf,28085934,1234195221815,3780779,119881519976,8217786,93786045792,9451947,146305093007,2421965,33566654314,20163811,20972584647,4215517,82058681794,2028 +12570,15000.0,1408997,19661286328,459996,5554383918,1181071,13245818324,1108549,5120904049,104639,788723905,1295034,322552761,119143,206009975,2029 +15000,20000.0,3211685,56540740564,674454,9272703298,2123856,24920098881,2217098,16010182775,240358,1989467760,2942317,1612763805,385402,1006505878,2029 +20000,30000.0,7739124,200103032447,995622,18129693871,2403583,27950921718,2755833,35552023516,446528,4496903457,5439143,3684635555,873371,3920075527,2029 +30000,40000.0,5563466,197776253000,616436,14549945400,1201792,13807081812,1502240,29312761111,377114,4473359460,3822941,2978315640,691030,5532839332,2029 +40000,50000.0,3626095,161711171571,371934,10081033598,588463,6757612399,807065,19895006537,336709,4449815463,2455385,2201363734,649589,9888478806,2029 +50000,70000.0,3553574,202429811895,322204,9492107237,456888,5230975711,666166,20483616198,411303,6356879232,2310341,2754647667,666166,14714998223,2029 +70000,100000.0,1699085,137279987377,148152,6189500583,182341,2121575986,268331,11148266977,234142,4438043464,1108549,1907063770,348105,11112766658,2029 +100000,150000.0,819498,95746974247,88062,5750692706,76666,950780349,112927,6392300917,141936,3001859637,506617,1483271821,229998,8087363023,2029 +150000,200000.0,273511,45604877162,41441,4214865136,21757,297469649,30045,2130766972,55945,1353779836,151260,765179908,95315,4061339509,2029 +200000,300000.0,186485,42580063881,35225,5069385738,14504,200932329,20721,1848234335,42477,1236059851,114999,862887496,77702,4355639474,2029 +300000,500000.0,96351,33738301982,21757,5589026646,8288,107762590,10360,1318485640,23829,900557891,70450,792255504,50765,4037795512,2029 +500000,1000000.0,45585,26874302614,15540,7702233001,3108,48268660,4144,753420366,13468,557992733,39369,768711507,33153,4496903457,2029 +1000000,inf,21757,41300335185,11396,22517772642,2072,22450540,2072,435571149,7252,516790738,21757,1459727824,19685,13066918422,2029 +12570,inf,28245211,1261347138253,3802220,124113343773,8264389,95661748947,9505550,150401540542,2435700,34560233426,20278162,21593376992,4239424,84487633796,2029 diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index dacd591bd..237edd7d7 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -1,80 +1,78 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 -afcs_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -alcohol_and_tobacco_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -benunit_rent,1.0,1.0,1.0,1.11,1.184,1.223,1.275,1.312,1.351,1.392,1.392,1.392,1.392,1.392,1.392 -bsp_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -capital_gains,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -capital_gains_before_response,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -carers_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -child_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -child_tax_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -diesel_spending,1.0,1.138,1.57,1.539,1.589,1.54,1.507,1.46,1.415,1.355,1.284,1.284,1.284,1.284,1.284 -dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -employee_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -employer_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -pension_contributions_via_salary_sacrifice,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -employment_income,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -employment_income_before_lsr,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -esa_contrib_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -esa_income_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -food_and_non_alcoholic_beverages_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -free_school_fruit_veg,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -free_school_meals,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -free_school_milk,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -gross_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -health_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -household_furnishings_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +afcs_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +alcohol_and_tobacco_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +bsp_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +bus_fare_spending,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +capital_gains,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +capital_gains_before_response,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +carers_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +child_benefit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +child_tax_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +childcare_expenses,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +clothing_and_footwear_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +communication_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +corporate_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +diesel_spending,1.0,1.13844,1.569717,1.539242,1.589317,1.539688,1.507377,1.459781,1.414994,1.35456,1.283802,1.283802,1.283802,1.283802,1.283802 +dividend_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +domestic_energy_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +education_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +employee_pension_contributions,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +employer_pension_contributions,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +employment_income,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +employment_income_before_lsr,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +esa_contrib_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +esa_income_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +food_and_non_alcoholic_beverages_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +free_school_fruit_veg,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +free_school_meals,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +free_school_milk,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +gross_financial_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +health_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +household_furnishings_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 household_weight,1.0,1.0,1.003,1.017,1.027,1.039,1.046,1.054,1.058,1.064,1.064,1.064,1.064,1.064,1.064 -housing_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -housing_service_charges,1.0,1.0,1.0,1.064,1.137,1.191,1.235,1.262,1.289,1.318,1.318,1.318,1.318,1.318,1.318 -housing_water_and_electricity_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -iidb_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -incapacity_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -income_support_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -jsa_contrib_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -jsa_income_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -lump_sum_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -main_residence_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -maintenance_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -maintenance_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -maternity_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -miscellaneous_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -miscellaneous_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -mortgage_capital_repayment,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -mortgage_interest_repayment,1.0,1.0,1.262,1.874,2.288,2.599,2.927,3.167,3.3,3.455,3.455,3.455,3.455,3.455,3.455 -net_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -non_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 -private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346 -private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -recreation_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -restaurants_and_hotels_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -savings,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -savings_interest_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -sda_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -self_employment_income,1.0,1.0,1.063,1.089,1.141,1.194,1.231,1.27,1.315,1.365,1.365,1.365,1.365,1.365,1.365 -state_pension,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -state_pension_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -statutory_maternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -statutory_paternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -statutory_sick_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -student_loan_repayments,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 -sublet_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 -transport_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -universal_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -water_and_sewerage_charges,1.0,1.0,1.0,1.092,1.14,1.21,1.283,1.349,1.4,1.46,1.46,1.46,1.46,1.46,1.46 -winter_fuel_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -working_tax_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +housing_benefit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +housing_water_and_electricity_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +iidb_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +incapacity_benefit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +income_support_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +jsa_contrib_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +jsa_income_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +lump_sum_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +main_residence_value,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +maintenance_expenses,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +maintenance_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +maternity_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +miscellaneous_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +miscellaneous_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +mortgage_capital_repayment,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +mortgage_interest_repayment,1.0,0.9774,1.120296,1.705538,2.17115,2.377844,2.567358,2.834363,2.969279,3.108241,3.280127,3.461518,3.65294,3.854948,4.068126 +net_financial_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +non_residential_property_value,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +other_investment_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +other_residential_property_value,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +owned_land,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +pension_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +pension_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +personal_pension_contributions,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +petrol_spending,1.0,1.104066,1.63457,1.795584,1.530966,1.483159,1.452034,1.406186,1.363043,1.304828,1.236668,1.236668,1.236668,1.236668,1.236668 +private_pension_income,1.0,1.003,1.05315,1.105808,1.161098,1.202781,1.252095,1.29091,1.329638,1.366867,1.406507,1.438856,1.47195,1.506835,1.542698 +private_pension_wealth,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +private_transfer_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +property_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +recreation_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +restaurants_and_hotels_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +savings,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +savings_interest_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +sda_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +self_employment_income,1.0,1.0635,1.09498,1.08841,1.118123,1.125838,1.160289,1.203336,1.247137,1.291161,1.337772,1.386066,1.436103,1.487946,1.541661 +state_pension,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +state_pension_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +statutory_maternity_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +statutory_paternity_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +statutory_sick_pay,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +student_loan_repayments,1.0,1.059,1.124023,1.193937,1.254828,1.321333,1.366259,1.399049,1.428429,1.459854,1.494891,1.544521,1.601823,1.661731,1.724379 +sublet_income,1.0,1.0892,1.200189,1.26404,1.311062,1.368486,1.408446,1.453939,1.499011,1.543382,1.591998,1.642146,1.693874,1.747231,1.802269 +transport_consumption,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +universal_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +winter_fuel_allowance_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 +working_tax_credit_reported,1.0,1.04,1.134328,1.217134,1.247562,1.289979,1.319649,1.346042,1.372963,1.400422,1.42843,1.456999,1.486139,1.515862,1.546179 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 122b4ed69..ae8b6a9bc 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -1,80 +1,78 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 -afcs_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -alcohol_and_tobacco_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -benunit_rent,0,0.0,0.0,0.11,0.067,0.033,0.043,0.029,0.03,0.03,0.0,0.0,0.0,0.0,0.0 -bsp_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -capital_gains,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -capital_gains_before_response,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -carers_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -child_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -child_tax_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -diesel_spending,0,0.138,0.379,-0.019,0.033,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 -dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -employee_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -employer_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -pension_contributions_via_salary_sacrifice,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -employment_income,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -employment_income_before_lsr,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -esa_contrib_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -esa_income_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -food_and_non_alcoholic_beverages_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -free_school_fruit_veg,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -free_school_meals,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -free_school_milk,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -gross_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -health_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -household_furnishings_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -household_weight,0,0.0,0.003,0.014,0.01,0.012,0.007,0.008,0.004,0.006,0.0,0.0,0.0,0.0,0.0 -housing_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -housing_service_charges,0,0.0,0.0,0.064,0.069,0.047,0.037,0.022,0.021,0.022,0.0,0.0,0.0,0.0,0.0 -housing_water_and_electricity_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -iidb_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -incapacity_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -income_support_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -jsa_contrib_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -jsa_income_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -lump_sum_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -main_residence_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -maintenance_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -maintenance_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -maternity_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -miscellaneous_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -miscellaneous_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -mortgage_capital_repayment,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -mortgage_interest_repayment,0,0.0,0.262,0.485,0.221,0.136,0.126,0.082,0.042,0.047,0.0,0.0,0.0,0.0,0.0 -net_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -non_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 -private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0 -private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -recreation_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -restaurants_and_hotels_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -savings,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -savings_interest_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -sda_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -self_employment_income,0,0.0,0.063,0.024,0.048,0.046,0.031,0.032,0.035,0.038,0.0,0.0,0.0,0.0,0.0 -state_pension,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -state_pension_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -statutory_maternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -statutory_paternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -statutory_sick_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -student_loan_repayments,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 -sublet_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 -transport_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -universal_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -water_and_sewerage_charges,0,0.0,0.0,0.092,0.044,0.061,0.06,0.051,0.038,0.043,0.0,0.0,0.0,0.0,0.0 -winter_fuel_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -working_tax_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +afcs_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +alcohol_and_tobacco_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +bsp_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +bus_fare_spending,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +capital_gains,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +capital_gains_before_response,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +carers_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +child_benefit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +child_tax_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +childcare_expenses,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +clothing_and_footwear_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +communication_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +corporate_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +diesel_spending,0,0.13844,0.378832,-0.019414,0.032532,-0.031227,-0.020985,-0.031575,-0.030681,-0.04271,-0.052237,0.0,0.0,0.0,0.0 +dividend_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +domestic_energy_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +education_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +employee_pension_contributions,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +employer_pension_contributions,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +employment_income,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +employment_income_before_lsr,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +esa_contrib_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +esa_income_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +food_and_non_alcoholic_beverages_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +free_school_fruit_veg,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +free_school_meals,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +free_school_milk,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +gross_financial_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +health_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +household_furnishings_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +household_weight,0,0.0,0.003,0.013958,0.009833,0.011685,0.006737,0.007648,0.003795,0.005671,0.0,0.0,0.0,0.0,0.0 +housing_benefit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +housing_water_and_electricity_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +iidb_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +incapacity_benefit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +income_support_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +jsa_contrib_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +jsa_income_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +lump_sum_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +main_residence_value,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +maintenance_expenses,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +maintenance_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +maternity_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +miscellaneous_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +miscellaneous_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +mortgage_capital_repayment,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +mortgage_interest_repayment,0,-0.0226,0.1462,0.522399,0.273,0.0952,0.0797,0.104,0.0476,0.0468,0.0553,0.0553,0.0553,0.0553,0.0553 +net_financial_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +non_residential_property_value,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +other_investment_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +other_residential_property_value,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +owned_land,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +pension_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +pension_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +personal_pension_contributions,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +petrol_spending,0,0.104066,0.4805,0.098505,-0.147372,-0.031227,-0.020986,-0.031575,-0.030681,-0.04271,-0.052237,0.0,0.0,0.0,0.0 +private_pension_income,0,0.003,0.05,0.05,0.05,0.0359,0.041,0.031,0.030001,0.027999,0.029001,0.023,0.023,0.0237,0.0238 +private_pension_wealth,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +private_transfer_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +property_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +recreation_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +restaurants_and_hotels_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +savings,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +savings_interest_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +sda_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +self_employment_income,0,0.0635,0.0296,-0.006,0.027299,0.0069,0.0306,0.0371,0.0364,0.0353,0.0361,0.0361,0.0361,0.0361,0.0361 +state_pension,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +state_pension_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +statutory_maternity_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +statutory_paternity_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +statutory_sick_pay,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +student_loan_repayments,0,0.059,0.0614,0.0622,0.051,0.052999,0.034001,0.024,0.021,0.022,0.024,0.0332,0.0371,0.0374,0.0377 +sublet_income,0,0.0892,0.1019,0.053201,0.0372,0.0438,0.0292,0.0323,0.031,0.0296,0.0315,0.0315,0.0315,0.0315,0.0315 +transport_consumption,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +universal_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +winter_fuel_allowance_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 +working_tax_credit_reported,0,0.04,0.0907,0.073,0.025,0.034,0.023,0.02,0.02,0.02,0.02,0.02,0.02,0.02,0.02 diff --git a/policyengine_uk_data/targets/compute/income.py b/policyengine_uk_data/targets/compute/income.py index f5be3e2bb..df97b7df7 100644 --- a/policyengine_uk_data/targets/compute/income.py +++ b/policyengine_uk_data/targets/compute/income.py @@ -1,8 +1,6 @@ """Income and salary sacrifice compute functions.""" import numpy as np -import pandas as pd -from policyengine_uk_data.storage import STORAGE_FOLDER def compute_income_band(target, ctx) -> np.ndarray: @@ -71,27 +69,21 @@ def compute_ss_ni_relief(target, ctx) -> np.ndarray: def compute_ss_headcount(target, ctx) -> np.ndarray: """Compute salary sacrifice user headcounts. - The 2k cap is defined at 2023-24 FRS base-year prices. The dataset - is uprated to 2025 for calibration then downrated to 2023 for - saving, but PE does not uprate SS when loading. To keep the - above/below classification consistent, deflate SS to base-year - prices before applying the threshold. + The 2k cap is applied to the contributions the calibration-year + simulation holds. uprating_factors.csv follows policyengine-uk's + load-time uprating, so these are the amounts the model runs on in that + year (the survey amounts, as policyengine-uk does not uprate this + variable at load), the values test_salary_sacrifice_headcount checks. """ ss = ctx.sim.calculate("pension_contributions_via_salary_sacrifice") - uprating = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index( - "Variable" - ) - row = "pension_contributions_via_salary_sacrifice" - price_adj = uprating.loc[row, "2023"] / uprating.loc[row, str(ctx.time_period)] - ss_base = ss * price_adj name = target.name if "below_cap" in name: - mask = (ss_base > 0) & (ss_base <= 2000) + mask = (ss > 0) & (ss <= 2000) elif "above_cap" in name: - mask = ss_base > 2000 + mask = ss > 2000 else: - mask = ss_base > 0 + mask = ss > 0 return ctx.household_from_person(mask) diff --git a/policyengine_uk_data/tests/test_income_projection.py b/policyengine_uk_data/tests/test_income_projection.py index 686604e06..6370578cc 100644 --- a/policyengine_uk_data/tests/test_income_projection.py +++ b/policyengine_uk_data/tests/test_income_projection.py @@ -143,3 +143,12 @@ def test_projection_keeps_top_open_ended_band(projections): ] assert len(top_band) == 1 assert top_band.iloc[0]["dividend_income_amount"] > 0 + + +def test_projection_is_the_spi_table_uprated_with_the_committed_table( + projections, base_targets +): + """Regenerate incomes_projection.csv whenever uprating_factors.csv changes.""" + from policyengine_uk_data.utils.incomes_projection import project_income_table + + pd.testing.assert_frame_equal(project_income_table(base_targets), projections) diff --git a/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py b/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py index e44a1ade9..77cf0f224 100644 --- a/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py +++ b/policyengine_uk_data/tests/test_road_fuel_volume_uprating.py @@ -32,6 +32,7 @@ END_YEAR, HOUSEHOLD_WEIGHT_UPRATING_INDEX, START_YEAR, + UPRATING_TABLE_DECIMALS, VOLUME_OVERRIDDEN_VARIABLES, _apply_household_weight_uprating_override, _apply_road_fuel_litre_proxy_override, @@ -109,7 +110,9 @@ def test__given_uprating_table__then_only_fuel_rows_are_overridden(): household_weight_index=df.loc["household_weight"], ) for year in range(START_YEAR, END_YEAR + 1): - assert out.loc[variable, year] == round(expected[year], 3) + assert out.loc[variable, year] == round( + expected[year], UPRATING_TABLE_DECIMALS + ) def test__given_generated_uprating_table__then_household_weight_row_is_restored(): @@ -144,7 +147,9 @@ def test__given_storage_csv__then_fuel_rows_reflect_litre_proxy_index(): ) assert variable in df.index for year in range(START_YEAR, END_YEAR + 1): - assert df.loc[variable, str(year)] == round(expected[year], 3) + assert df.loc[variable, str(year)] == round( + expected[year], UPRATING_TABLE_DECIMALS + ) def test__given_storage_csv__then_household_weight_row_is_unchanged(): diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py b/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py index cbae09725..4fcf007fb 100644 --- a/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py +++ b/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py @@ -6,8 +6,12 @@ """ import os +from types import SimpleNamespace + +import numpy as np from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.targets.compute.income import compute_ss_headcount # The total combines below-cap and above-cap users and moves slightly with # each generated FRS calibration refresh. Widened from 0.16 after the @@ -87,3 +91,40 @@ def test_salary_sacrifice_above_cap_users(baseline): f"Expected ~{TARGET / 1e6:.1f}mn above-cap SS users, " f"got {total_above_cap / 1e6:.1f}mn ({total_above_cap / TARGET * 100:.0f}% of target)" ) + + +def _headcount_masks(contributions): + """Run compute_ss_headcount for each target on one person per household.""" + contributions = np.asarray(contributions, dtype=float) + ctx = SimpleNamespace( + sim=SimpleNamespace(calculate=lambda variable: contributions), + household_from_person=lambda values: np.asarray(values), + time_period=PERIOD, + ) + return { + kind: compute_ss_headcount( + SimpleNamespace(name=f"obr/salary_sacrifice_users_{kind}"), ctx + ) + for kind in ("total", "below_cap", "above_cap") + } + + +def test_headcount_targets_split_users_at_the_cap_on_simulated_amounts(): + """Calibration classifies the amounts the calibration-year simulation holds. + + The same amounts the tests above classify, with no adjustment through the + uprating table (which has no salary sacrifice row, as policyengine-uk does + not uprate it at load). Below and above the cap partition the users. + """ + edges = [0.0, 0.01, 1_999.99, 2_000.0, 2_000.01, 1e6] + rng = np.random.default_rng(0) + for contributions in (edges, rng.uniform(0, 5_000, 1_000)): + masks = _headcount_masks(contributions) + contributions = np.asarray(contributions) + np.testing.assert_array_equal( + masks["below_cap"], (contributions > 0) & (contributions <= 2_000) + ) + np.testing.assert_array_equal(masks["above_cap"], contributions > 2_000) + np.testing.assert_array_equal( + masks["below_cap"].astype(int) + masks["above_cap"], masks["total"] + ) diff --git a/policyengine_uk_data/tests/test_uprating_factors_table.py b/policyengine_uk_data/tests/test_uprating_factors_table.py new file mode 100644 index 000000000..dec1185ef --- /dev/null +++ b/policyengine_uk_data/tests/test_uprating_factors_table.py @@ -0,0 +1,225 @@ +"""uprating_factors.csv follows policyengine-uk's load-time uprating. + +``uprate_dataset`` moves the build between the FRS base year and the +calibration year with this table, and policyengine-uk moves the saved file +to later years with ``extend_single_year_dataset``. Calibrated values are the +values the model runs on only where the two agree. + +Invariants, checked for every row and every year (or pair of years): +- the committed table is what the generator builds from the locked + policyengine-uk, and its rows are exactly the variables policyengine-uk + uprates at load; +- outside ``OVERRIDDEN_VARIABLES`` each row grows each year by one plus the + growth parameter policyengine-uk applies to it, and agrees with + policyengine-uk's own ``extend_single_year_dataset`` (differential), as + does ``uprate_dataset`` from the FRS base year; +- the fuel rows divide by the household-weight override, so fuel spending + times household weight stays on policyengine-uk's path; +- every row is a level index (1 in ``START_YEAR``, finite, above 0.5), so + ``uprate_dataset`` there and back between any two years is the identity; +- every variable a target projection uprates has a row. + +test_income_projection checks that incomes_projection.csv is the committed +SPI table projected with this table. +""" + +import numpy as np +import pandas as pd +import pytest +from policyengine_uk.data import UKSingleYearDataset + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.utils import uprating +from policyengine_uk_data.utils.uprating import ( + END_YEAR, + OVERRIDDEN_VARIABLES, + START_YEAR, + UPRATING_TABLE_DECIMALS, + VOLUME_OVERRIDDEN_VARIABLES, + build_uprating_factors_table, + policyengine_uk_load_time_index, + policyengine_uk_uprating_indices, + uprate_dataset, +) + +TABLE = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv", index_col="Variable") +TABLE.columns = TABLE.columns.astype(int) +YEARS = list(range(START_YEAR, END_YEAR + 1)) +ENGINE_ROWS = TABLE.index.difference(OVERRIDDEN_VARIABLES) +# Each stored level is within half a unit in the last stored place of the +# unrounded level, and every level exceeds 0.5, so a ratio of two stored +# levels is within 2 units in that place, relative, of the unrounded ratio. +ROUNDING = 2 * 10**-UPRATING_TABLE_DECIMALS +# The engine's own uprating moves council tax by country and rent by region +# and tenure. Neither is a single index, so neither is in the table and +# `uprate_dataset` leaves both unchanged. +NOT_SINGLE_INDICES = ("council_tax", "rent") +# Dataset columns that declare an `uprating` attribute policyengine-uk does not +# apply at load (policyengine-uk#1862): both paths carry them unchanged. +CARRIED_UNCHANGED = ( + "housing_service_charges", + "pension_contributions_via_salary_sacrifice", + "rail_usage", + "water_and_sewerage_charges", +) + + +def _unit_dataset(variables, year: int) -> UKSingleYearDataset: + """One person, benefit unit and household with every variable at 1.""" + from policyengine_uk.system import system + + tables = { + "person": { + "person_id": [1], + "person_benunit_id": [1], + "person_household_id": [1], + }, + "benunit": {"benunit_id": [1]}, + "household": { + "household_id": [1], + "region": ["LONDON"], + "tenure_type": ["RENT_PRIVATELY"], + **{variable: [1.0] for variable in NOT_SINGLE_INDICES}, + }, + } + for variable in variables: + tables[system.variables[variable].entity.key][variable] = [1.0] + return UKSingleYearDataset( + **{name: pd.DataFrame(columns) for name, columns in tables.items()}, + fiscal_year=year, + ) + + +def _value(dataset: UKSingleYearDataset, variable: str) -> float: + (column,) = [table[variable] for table in dataset.tables if variable in table] + return float(column.iloc[0]) + + +def _project_with_policyengine_uk(dataset, end_year: int) -> dict: + from policyengine_uk.data.economic_assumptions import extend_single_year_dataset + from policyengine_uk.system import system + + return extend_single_year_dataset( + dataset, system.parameters, end_year=end_year + ).datasets + + +def test_committed_table_is_what_the_generator_builds(): + """Fails when policyengine-uk's growth parameters or indices change. + + Regenerate with ``python -m policyengine_uk_data.utils.uprating``, then + ``python -m policyengine_uk_data.utils.incomes_projection``. + """ + pd.testing.assert_frame_equal( + TABLE, build_uprating_factors_table(), check_names=False, rtol=0, atol=1e-12 + ) + + +def test_rows_are_the_variables_policyengine_uk_uprates_at_load(): + engine = policyengine_uk_load_time_index() + assert set(TABLE.index) == set(engine.index) + assert set(OVERRIDDEN_VARIABLES) <= set(TABLE.index) + + +def test_each_row_grows_by_its_policyengine_uk_growth_parameter(): + from policyengine_uk.system import system + + for index_name, variables in policyengine_uk_uprating_indices().items(): + growth = system.parameters.get_child(index_name) + for variable in set(variables).difference(OVERRIDDEN_VARIABLES): + for year in YEARS[1:]: + factor = TABLE.loc[variable, year] / TABLE.loc[variable, year - 1] + assert factor == pytest.approx(1 + growth(str(year)), rel=ROUNDING), ( + variable, + year, + ) + + +def test_table_agrees_with_policyengine_uks_own_load_time_uprating(): + """Differential: policyengine-uk projects a dataset of ones from START_YEAR.""" + projected = _project_with_policyengine_uk( + _unit_dataset(TABLE.index, START_YEAR), END_YEAR + ) + for year in YEARS: + engine = projected[year] + for variable in ENGINE_ROWS: + assert TABLE.loc[variable, year] == pytest.approx( + _value(engine, variable), rel=ROUNDING + ), (variable, year) + engine_weight = _value(engine, "household_weight") + for variable in VOLUME_OVERRIDDEN_VARIABLES: + weighted = TABLE.loc[variable, year] * TABLE.loc["household_weight", year] + assert weighted == pytest.approx( + _value(engine, variable) * engine_weight, rel=ROUNDING + ), (variable, year) + + +@pytest.mark.parametrize("year", range(CURRENT_FRS_RELEASE.base_year, 2031)) +def test_uprate_dataset_from_the_base_year_matches_policyengine_uk(year): + """What calibration sees in ``year`` is what policyengine-uk runs on. + + policyengine-uk projects a dataset to 2030 at load, so later years follow + each variable's own ``uprating`` attribute and are not compared. + """ + base_year = CURRENT_FRS_RELEASE.base_year + variables = list(TABLE.index) + list(CARRIED_UNCHANGED) + dataset = _unit_dataset(variables, base_year) + engine = _project_with_policyengine_uk(dataset, year)[year] + calibration = uprate_dataset(dataset, year) + for variable in ENGINE_ROWS.union(CARRIED_UNCHANGED): + assert _value(calibration, variable) == pytest.approx( + _value(engine, variable), rel=ROUNDING + ), variable + for variable in CARRIED_UNCHANGED: + assert _value(calibration, variable) == 1.0, variable + for variable in VOLUME_OVERRIDDEN_VARIABLES: + assert _value(calibration, variable) * _value( + calibration, "household_weight" + ) == pytest.approx( + _value(engine, variable) * _value(engine, "household_weight"), + rel=ROUNDING, + ), variable + + +def test_every_row_is_a_level_index(): + levels = TABLE.to_numpy() + assert (TABLE[START_YEAR] == 1).all() + assert np.isfinite(levels).all() + assert (levels > 0.5).all() + + +@pytest.mark.parametrize("start", YEARS) +def test_uprate_dataset_there_and_back_is_the_identity(start): + dataset = _unit_dataset(TABLE.index, start) + for end in YEARS: + there = uprate_dataset(dataset, end) + back = uprate_dataset(there, start) + for variable in TABLE.index: + assert _value(there, variable) == pytest.approx( + TABLE.loc[variable, end] / TABLE.loc[variable, start], rel=1e-15 + ) + assert _value(back, variable) == pytest.approx(1.0, rel=1e-12) + + +def test_every_variable_a_target_projection_uprates_has_a_row(): + """A missing row loses targets: the CGT projection stops at the first year + it cannot uprate, behind a log warning, and the SPI income projection + cannot be regenerated.""" + from policyengine_uk_data.utils.incomes_projection import ALL_INCOME_VARIABLES + + needed = set(ALL_INCOME_VARIABLES) | {"household_weight", "capital_gains"} + assert needed <= set(TABLE.index), needed - set(TABLE.index) + + +def test_a_variable_listed_under_two_indices_raises(monkeypatch): + monkeypatch.setattr( + uprating, + "policyengine_uk_uprating_indices", + lambda: { + "gov.economic_assumptions.yoy_growth.obr.consumer_price_index": ["x"], + "gov.economic_assumptions.yoy_growth.obr.average_earnings": ["x"], + }, + ) + with pytest.raises(ValueError, match="listed under two indices"): + policyengine_uk_load_time_index() diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index ff3939fb1..92eb83479 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -1,9 +1,15 @@ +from pathlib import Path + from policyengine_uk_data.storage import STORAGE_FOLDER import pandas as pd +import yaml from policyengine_uk.data import UKSingleYearDataset START_YEAR = 2020 END_YEAR = 2034 +# Decimal places stored in uprating_factors.csv. At 3 places the rounding +# alone moved the factor between two years by up to 0.08%. +UPRATING_TABLE_DECIMALS = 6 # These variables are named as spending, but PolicyEngine UK derives fuel # litres through ``litres = spending / price`` and uprates household weights @@ -31,6 +37,10 @@ 2033: 1.064, 2034: 1.064, } +# Rows of the table that deliberately differ from policyengine-uk's own +# index. The fuel rows divide by the household-weight row, so together they +# keep weighted fuel spending on the engine's path. +OVERRIDDEN_VARIABLES = ("household_weight",) + VOLUME_OVERRIDDEN_VARIABLES class UpratingYearOutOfRangeError(ValueError): @@ -55,32 +65,67 @@ def _check_year_in_range(year: int, *, kind: str) -> None: ) -def create_policyengine_uprating_factors_table(): - from policyengine_uk.system import system +def policyengine_uk_uprating_indices() -> dict[str, list[str]]: + """The growth index policyengine-uk projects each dataset variable by. - df = pd.DataFrame() + Read from the ``uprating_indices.yaml`` that + ``policyengine_uk.data.economic_assumptions`` loads: a mapping from a + year-on-year growth parameter to the variables it uprates. + """ + from policyengine_uk.data import economic_assumptions + + path = Path(economic_assumptions.__file__).with_name("uprating_indices.yaml") + with open(path) as f: + return yaml.safe_load(f) + + +def policyengine_uk_load_time_index( + start_year: int = START_YEAR, end_year: int = END_YEAR +) -> pd.DataFrame: + """Level index (``start_year`` = 1) that policyengine-uk applies to data. + + policyengine-uk projects a single-year dataset to later years with + ``economic_assumptions.extend_single_year_dataset`` (to 2030 by default): + each year, every variable listed in ``uprating_indices.yaml`` is + multiplied by one plus the year-on-year growth parameter it is listed + under. Variables the file does not list are carried forward unchanged, + and a variable's own ``uprating`` attribute only applies after the last + year the dataset is projected to, so neither appears here; past that year + this index keeps compounding the same growth parameters. Council tax and + rent, which the engine uprates by country and region, are not single + indices and are not covered. + """ + from policyengine_uk.system import system - variable_names = [] - years = [] - index_values = [] + rows = {} + for index_name, variables in policyengine_uk_uprating_indices().items(): + growth = system.parameters.get_child(index_name) + level = [1.0] + for year in range(start_year + 1, end_year + 1): + level.append(level[-1] * (1 + growth(str(year)))) + for variable in variables: + if variable in rows: + raise ValueError( + f"{variable} is listed under two indices in " + "policyengine-uk's uprating_indices.yaml." + ) + rows[variable] = level + df = pd.DataFrame.from_dict( + rows, orient="index", columns=range(start_year, end_year + 1) + ) + df.index.name = "Variable" + return df.sort_index() - for variable in system.variables.values(): - if variable.uprating is not None: - parameter = system.parameters.get_child(variable.uprating) - start_value = parameter(START_YEAR) - for year in range(START_YEAR, END_YEAR + 1): - variable_names.append(variable.name) - years.append(year) - growth = parameter(year) / start_value - index_values.append(round(growth, 3)) - df["Variable"] = variable_names - df["Year"] = years - df["Value"] = index_values +def build_uprating_factors_table() -> pd.DataFrame: + """The uprating factor table: policyengine-uk's load-time index, overridden. - # Convert to there is a column for each year - df = df.pivot(index="Variable", columns="Year", values="Value") - df = df.sort_values("Variable") + ``uprate_dataset`` moves the build between the FRS base year and the + calibration year with this table, and policyengine-uk moves the saved file + with its own load-time index, so the two must agree for calibrated values + to be the values the model runs on. Only ``OVERRIDDEN_VARIABLES`` differ. + """ + df = policyengine_uk_load_time_index().round(UPRATING_TABLE_DECIMALS) # Keep the population calibration row stable. Current PolicyEngine UK # population indices would inflate final calibrated population above the @@ -90,14 +135,20 @@ def create_policyengine_uprating_factors_table(): # Ensure petrol/diesel use sourced road-fuel clearances and model pump # prices. This keeps litres aligned after PolicyEngine divides by price. df = _apply_road_fuel_litre_proxy_override(df) + return df + +def create_policyengine_uprating_factors_table(): + df = build_uprating_factors_table() df.to_csv(STORAGE_FOLDER / "uprating_factors.csv") # Create a table with growth factors by year df_growth = df.copy() for year in range(END_YEAR, START_YEAR, -1): - df_growth[year] = round(df_growth[year] / df_growth[year - 1] - 1, 3) + df_growth[year] = round( + df_growth[year] / df_growth[year - 1] - 1, UPRATING_TABLE_DECIMALS + ) df_growth[START_YEAR] = 0 df_growth.to_csv(STORAGE_FOLDER / "uprating_growth_factors.csv") @@ -224,7 +275,7 @@ def _apply_road_fuel_litre_proxy_override(df: pd.DataFrame) -> pd.DataFrame: + ", ".join(str(year) for year in missing_years) ) for year in range(START_YEAR, END_YEAR + 1): - df.loc[variable, year] = round(index[year], 3) + df.loc[variable, year] = round(index[year], UPRATING_TABLE_DECIMALS) return df From 3c37ef808fedb0c8abced657a66d2c12bcf275ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 18:16:14 -0400 Subject: [PATCH 09/21] Probe the imported model for unknown-region support; tighten SPI tests Review of #540 round 1 (Sol 6.1, REQUEST CHANGES): - model_simulates_unknown_region now simulates one UNKNOWN household on the imported policyengine-uk instead of reading installed package metadata, which differs from the imported code under make data-local. Only a missing parameter for UNKNOWN reads as "cannot simulate"; any other error raises. - A differential test ties the probe to the release: where the imported model is the installed distribution, the probe agrees with version >= 2.104.5. - The unknown-region test xfails only on the missing private rent index for UNKNOWN; any other error fails it. - The region invariant test checks HMRC's codebook mapping written out in the test, not REGION_MAP itself. - New test: records whose SCOT_TXP agrees with their region keep exactly the income tax of the region-derived rule; the others change. - Income tax tests now cover 2030, the last year policyengine-uk carries dataset inputs to; the create_spi comment records that horizon. Co-Authored-By: Claude Opus 5.5 --- changelog.d/spi-unknown-region.fixed.md | 2 +- policyengine_uk_data/datasets/spi.py | 54 +++++-- policyengine_uk_data/tests/test_spi_build.py | 155 +++++++++++++++---- 3 files changed, 174 insertions(+), 37 deletions(-) diff --git a/changelog.d/spi-unknown-region.fixed.md b/changelog.d/spi-unknown-region.fixed.md index a8a4b44cf..f5111f98e 100644 --- a/changelog.d/spi-unknown-region.fixed.md +++ b/changelog.d/spi-unknown-region.fixed.md @@ -1 +1 @@ -Take Scottish taxpayer status in the SPI dataset from HMRC's `SCOT_TXP` flag rather than the region code. `GORCODE` 13, 14 and -1 (address abroad, address unknown, composite records) stay `UNKNOWN`, which policyengine-uk 2.104.5 and later can simulate; `load_spi_dataset` relabels them `SOUTH_EAST` only for older releases, and rebuilds a cached SPI H5 that predates the flag. +Take Scottish taxpayer status in the SPI dataset from HMRC's `SCOT_TXP` flag rather than the region code. `GORCODE` 13, 14 and -1 (address abroad, address unknown, composite records) stay `UNKNOWN`, which policyengine-uk 2.104.5 and later can simulate; `load_spi_dataset` relabels them `SOUTH_EAST` only when the imported model cannot simulate them, and rebuilds a cached SPI H5 that predates the flag. diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index a21ddb14f..9f661f8bb 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -1,5 +1,4 @@ -import re -from importlib.metadata import version +from functools import cache from policyengine_uk_data.storage import STORAGE_FOLDER import pandas as pd @@ -46,15 +45,49 @@ 12: "NORTHERN_IRELAND", } -# First policyengine-uk release that can simulate Region.UNKNOWN: it uprates -# their rent by the UK-wide index (PolicyEngine/policyengine-uk#1985). -UNKNOWN_REGION_MODEL_VERSION = (2, 104, 5) - +@cache def model_simulates_unknown_region() -> bool: - """Whether the installed policyengine-uk can simulate Region.UNKNOWN.""" - release = re.match(r"\d+(\.\d+)*", version("policyengine-uk")).group() - return tuple(map(int, release.split("."))) >= UNKNOWN_REGION_MODEL_VERSION + """Whether the imported policyengine-uk can simulate Region.UNKNOWN. + + Releases before 2.104.5 have no rent index for it + (PolicyEngine/policyengine-uk#1985). This simulates one household rather + than reading the installed version, which need not be the imported code + (``make data-local`` puts a checkout on PYTHONPATH). + """ + from policyengine_core.errors import ParameterNotFoundError + from policyengine_uk import Microsimulation + + ids = [1] + household = UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": ids, + "person_benunit_id": ids, + "person_household_id": ids, + "age": [40], + } + ), + benunit=pd.DataFrame({"benunit_id": ids}), + household=pd.DataFrame( + { + "household_id": ids, + "household_weight": [1.0], + "region": ["UNKNOWN"], + "rent": [0.0], + "tenure_type": ["OWNED_OUTRIGHT"], + "council_tax": [0.0], + } + ), + fiscal_year=SPI_FISCAL_YEAR, + ) + try: + Microsimulation(dataset=household) + except ParameterNotFoundError as error: + if ".UNKNOWN'" not in str(error): + raise + return False + return True def _get_marriage_allowance(fiscal_year: int) -> float: @@ -138,7 +171,8 @@ def create_spi( # disagree for some records and SCOT_TXP is also set on records with no # UK region, so SCOT_TXP decides the rates. HMRC documents it as "." or 1; # the 2022-23 tape holds 0 or 1. WELSH_TXP is not used: policyengine-uk - # has no separate Welsh rates. + # has no separate Welsh rates. policyengine-uk carries dataset inputs to + # 2030; from 2031 it derives the status from the region again. person["pays_scottish_income_tax"] = ( pd.to_numeric(df.SCOT_TXP, errors="coerce") == 1 ) diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index 14c4a5c41..a6a26109a 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -23,7 +23,6 @@ import numpy as np import pandas as pd import pytest -from policyengine_core.errors import ParameterNotFoundError if importlib.util.find_spec("policyengine_uk") is None: pytest.skip( @@ -31,6 +30,8 @@ allow_module_level=True, ) +from policyengine_core.errors import ParameterNotFoundError + from policyengine_uk_data.datasets.spi import model_simulates_unknown_region @@ -529,6 +530,24 @@ def _set_spi_columns(path, **columns): df.to_csv(path, sep="\t", index=False) +# HMRC's GORCODE codes 1-12 (SN 9422, Annex A) as policyengine-uk regions, +# written out here rather than read from REGION_MAP so a wrong mapping fails. +HMRC_REGIONS = { + 1: "NORTH_EAST", + 2: "NORTH_WEST", + 3: "YORKSHIRE", # Yorkshire and the Humber + 4: "EAST_MIDLANDS", + 5: "WEST_MIDLANDS", + 6: "EAST_OF_ENGLAND", + 7: "LONDON", + 8: "SOUTH_EAST", + 9: "SOUTH_WEST", + 10: "WALES", + 11: "SCOTLAND", + 12: "NORTHERN_IRELAND", +} + + @pytest.mark.parametrize("not_scottish", [0, ".", ""]) def test_create_spi_region_and_scottish_taxpayer_invariants(tmp_path, not_scottish): """For every GORCODE (documented 1-14, composite -1, undocumented 99) and @@ -540,7 +559,7 @@ def test_create_spi_region_and_scottish_taxpayer_invariants(tmp_path, not_scotti from policyengine_uk.variables.household.demographic.geography import Region - from policyengine_uk_data.datasets.spi import REGION_MAP, create_spi + from policyengine_uk_data.datasets.spi import create_spi cases = list(product([-1, *range(1, 15), 99], (not_scottish, 1))) gor = [g for g, _ in cases] @@ -552,7 +571,7 @@ def test_create_spi_region_and_scottish_taxpayer_invariants(tmp_path, not_scotti ds = create_spi(tab, 2022) regions = ds.household["region"].tolist() - assert regions == [REGION_MAP.get(g, "UNKNOWN") for g in gor] + assert regions == [HMRC_REGIONS.get(g, "UNKNOWN") for g in gor] assert set(regions) <= {region.name for region in Region} assert ds.person["pays_scottish_income_tax"].tolist() == [s == 1 for s in scot] # Address abroad, address unknown and composite records stay UNKNOWN even @@ -577,12 +596,74 @@ def test_create_spi_scottish_taxpayer_status_survives_h5_round_trip(tmp_path): assert loaded.household["region"].tolist() == ["UNKNOWN", "SCOTLAND", "LONDON"] +UNKNOWN_RENT_INDEX = ( + "gov.economic_assumptions.yoy_growth.ons.private_rental_prices.UNKNOWN" +) + + +@pytest.mark.parametrize( + "missing, expected", [(None, True), (UNKNOWN_RENT_INDEX, False)] +) +def test_unknown_region_probe_reads_the_simulation(monkeypatch, missing, expected): + import policyengine_uk + + def simulate(dataset): + assert dataset.household["region"].tolist() == ["UNKNOWN"] + if missing: + raise ParameterNotFoundError(missing, "2023-01-01") + + monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) + + assert model_simulates_unknown_region.__wrapped__() is expected + + +def test_unknown_region_probe_raises_other_missing_parameters(monkeypatch): + import policyengine_uk + + def simulate(dataset): + raise ParameterNotFoundError("gov.hmrc.income_tax.rates.uk", "2023-01-01") + + monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) + + with pytest.raises(ParameterNotFoundError, match=r"rates\.uk'"): + model_simulates_unknown_region.__wrapped__() + + +def test_unknown_region_probe_agrees_with_policyengine_uk_release(): + """2.104.5 is the first policyengine-uk release with + PolicyEngine/policyengine-uk#1985. Where the imported model is the + installed release, the probe agrees with its version.""" + from importlib.metadata import PackageNotFoundError, distribution + from pathlib import Path + + import policyengine_uk + from packaging.version import Version + + try: + release = distribution("policyengine-uk") + except PackageNotFoundError: + pytest.skip("policyengine-uk is not installed as a distribution") + installed = Path(release.locate_file("policyengine_uk/__init__.py")) + if not installed.exists() or not installed.samefile(policyengine_uk.__file__): + pytest.skip("the imported policyengine-uk is not the installed release") + + assert model_simulates_unknown_region() == ( + Version(release.version) >= Version("2.104.5") + ) + + # GORCODE, SCOT_TXP: abroad, unknown, composite, London, Scotland, abroad and # Scottish, Scotland but not Scottish. SIMULATED_RECORDS = ((13, 0), (14, 0), (-1, 0), (7, 0), (11, 1), (13, 1), (11, 0)) +# The data year, an uprated year and 2030, the last year policyengine-uk +# carries dataset inputs to. +YEARS = [2022, 2026, 2030] -def _spi_income_tax(tmp_path, years, **kwargs): +def _spi_income_tax(tmp_path, region_rule=False, **kwargs): + """Income tax on SIMULATED_RECORDS, each paid £60,000, in YEARS. With + region_rule, the Scottish taxpayer flag is dropped, so policyengine-uk + derives it from the region as it did before create_spi read SCOT_TXP.""" from policyengine_uk import Microsimulation from policyengine_uk_data.datasets.spi import create_spi @@ -593,40 +674,62 @@ def _spi_income_tax(tmp_path, years, **kwargs): _set_spi_columns( tab, SCOT_TXP=scot, PAY=[60_000] * len(gor), AGERANGE=[3] * len(gor) ) - sim = Microsimulation(dataset=create_spi(tab, 2022, **kwargs)) - return {year: sim.calculate("income_tax", year).values for year in years} + dataset = create_spi(tab, 2022, **kwargs) + if region_rule: + dataset.person = dataset.person.drop(columns="pays_scottish_income_tax") + sim = Microsimulation(dataset=dataset) + return {year: sim.calculate("income_tax", year).values for year in YEARS} -@pytest.mark.parametrize("year", [2022, 2026]) -def test_spi_income_tax_follows_scottish_taxpayer_flag(tmp_path, year): +def test_spi_income_tax_follows_scottish_taxpayer_flag(tmp_path): """Equal pay, so income tax depends only on SCOT_TXP, in the data year and after uprating. Uses the legacy SOUTH_EAST label so it runs on any policyengine-uk release.""" - tax = _spi_income_tax(tmp_path, [year], unknown_region="SOUTH_EAST")[year] - abroad, unknown, composite, london, scotland, abroad_scot, scotland_ruk = tax + for year, tax in _spi_income_tax(tmp_path, unknown_region="SOUTH_EAST").items(): + abroad, unknown, composite, london, scotland, abroad_scot, scotland_ruk = tax - assert london > 0 - assert abroad == unknown == composite == scotland_ruk == london - assert abroad_scot == scotland != london + assert london > 0, year + assert abroad == unknown == composite == scotland_ruk == london, year + assert abroad_scot == scotland != london, year + + +def test_spi_income_tax_changes_only_where_scottish_flag_and_region_disagree( + tmp_path, +): + """Records whose SCOT_TXP agrees with their region get exactly the income + tax they got when the region decided Scottish status. The two records + where they disagree change.""" + label = "UNKNOWN" if model_simulates_unknown_region() else "SOUTH_EAST" + flag = _spi_income_tax(tmp_path, unknown_region=label) + region = _spi_income_tax(tmp_path, region_rule=True, unknown_region=label) + agrees = np.array([(g == 11) == (s == 1) for g, s in SIMULATED_RECORDS]) + + for year in YEARS: + assert (flag[year][agrees] == region[year][agrees]).all(), year + assert (flag[year][~agrees] != region[year][~agrees]).all(), year -@pytest.mark.xfail( - condition=not model_simulates_unknown_region(), - reason="policyengine-uk before 2.104.5 (PolicyEngine/policyengine-uk#1985) " - "has no rent index for Region.UNKNOWN", - raises=ParameterNotFoundError, - strict=True, -) def test_create_spi_output_with_unknown_region_can_be_simulated(tmp_path): """SPI records with an address abroad (13), an unknown address (14) or a composite record (-1) keep region UNKNOWN and still run through policyengine-uk, giving record for record the income tax of the legacy SOUTH_EAST relabelling: the region label does not move income tax. + Models without PolicyEngine/policyengine-uk#1985 must fail on the missing + rent index, and on nothing else. """ - years = [2022, 2026] - tax = _spi_income_tax(tmp_path, years) - legacy = _spi_income_tax(tmp_path, years, unknown_region="SOUTH_EAST") + if not model_simulates_unknown_region(): + with pytest.raises( + ParameterNotFoundError, match=r"private_rental_prices\.UNKNOWN'" + ): + _spi_income_tax(tmp_path) + pytest.xfail( + "policyengine-uk before 2.104.5 (PolicyEngine/policyengine-uk#1985) " + "has no rent index for Region.UNKNOWN" + ) + + tax = _spi_income_tax(tmp_path) + legacy = _spi_income_tax(tmp_path, unknown_region="SOUTH_EAST") - for year in years: - assert (tax[year] > 0).all() - assert (tax[year] == legacy[year]).all() + for year in YEARS: + assert (tax[year] > 0).all(), year + assert (tax[year] == legacy[year]).all(), year From cc5b4c168320e631b91ca6f58438ef1e5933b12d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 18:50:08 -0400 Subject: [PATCH 10/21] Pin the load-time changes the uprating table does not cover Review r1 of #541: policyengine-uk also reassigns student loan plans at load and zeroes the repayments of loans it writes off, which the table's single average-earnings index cannot represent. The generator docstring and the test module now name it beside council tax and rent, and a test projects a kept Plan 2 loan and a written-off Plan 1 loan through both paths: repayments agree for the first, and only the engine zeroes the second. The docstring also states when policyengine-core applies a variable's `uprating` attribute: only to periods with no stored value. A second test fails if the table gains a salary sacrifice row, as policyengine-uk#1863 would add. The headcount targets then move from survey-year to nominal calibration-year contributions, a methodology choice to make at that point. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_salary_sacrifice_headcount.py | 15 ++++++ .../tests/test_uprating_factors_table.py | 50 +++++++++++++++++++ policyengine_uk_data/utils/uprating.py | 12 +++-- 3 files changed, 72 insertions(+), 5 deletions(-) diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py b/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py index 4fcf007fb..404d2aef8 100644 --- a/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py +++ b/policyengine_uk_data/tests/test_salary_sacrifice_headcount.py @@ -9,8 +9,10 @@ from types import SimpleNamespace import numpy as np +import pandas as pd from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.storage import STORAGE_FOLDER from policyengine_uk_data.targets.compute.income import compute_ss_headcount # The total combines below-cap and above-cap users and moves slightly with @@ -128,3 +130,16 @@ def test_headcount_targets_split_users_at_the_cap_on_simulated_amounts(): np.testing.assert_array_equal( masks["below_cap"].astype(int) + masks["above_cap"], masks["total"] ) + + +def test_calibration_year_contributions_are_the_survey_amounts(): + """The cap split classifies survey-year contributions only while + policyengine-uk carries them unchanged at load, so the table has no row. + + If policyengine-uk starts uprating them (policyengine-uk#1863), the + regenerated table gains a row and the split moves to nominal + calibration-year amounts. Decide then which of the two the headcount + targets should classify (policyengine-uk-data#541). + """ + table = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv", index_col="Variable") + assert "pension_contributions_via_salary_sacrifice" not in table.index diff --git a/policyengine_uk_data/tests/test_uprating_factors_table.py b/policyengine_uk_data/tests/test_uprating_factors_table.py index dec1185ef..a17f24d28 100644 --- a/policyengine_uk_data/tests/test_uprating_factors_table.py +++ b/policyengine_uk_data/tests/test_uprating_factors_table.py @@ -19,6 +19,10 @@ ``uprate_dataset`` there and back between any two years is the identity; - every variable a target projection uprates has a row. +The engine's other load-time changes are outside the table: council tax by +country, rent by region and tenure, and student loan plan reassignment, which +zeroes the repayments of loans it writes off. + test_income_projection checks that incomes_projection.csv is the committed SPI table projected with this table. """ @@ -182,6 +186,52 @@ def test_uprate_dataset_from_the_base_year_matches_policyengine_uk(year): ), variable +def test_student_loan_plan_reassignment_is_outside_the_table(): + """The table grows repayments by average earnings, as the engine does for a + loan it keeps. It does not zero a loan the engine writes off at load.""" + base_year = CURRENT_FRS_RELEASE.base_year + year = base_year + 1 + dataset = UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [1, 2], + "person_benunit_id": [1, 2], + "person_household_id": [1, 1], + # University starts in 2012 (Plan 2) and 1982 (written off). + "age": [base_year - 2012 + 18.0, base_year - 1982 + 18.0], + "student_loan_plan": ["PLAN_2", "PLAN_1"], + "highest_education": ["TERTIARY", "TERTIARY"], + "student_loan_repayments": [1.0, 1.0], + } + ), + benunit=pd.DataFrame({"benunit_id": [1, 2]}), + household=pd.DataFrame( + { + "household_id": [1], + "region": ["LONDON"], + "tenure_type": ["RENT_PRIVATELY"], + **{variable: [1.0] for variable in NOT_SINGLE_INDICES}, + } + ), + fiscal_year=base_year, + ) + engine = _project_with_policyengine_uk(dataset, year)[year].person + calibration = uprate_dataset(dataset, year).person + growth = ( + TABLE.loc["student_loan_repayments", year] + / TABLE.loc["student_loan_repayments", base_year] + ) + assert list(engine.student_loan_plan) == ["PLAN_2", "NONE"] + assert engine.student_loan_repayments.tolist() == [ + pytest.approx(growth, rel=ROUNDING), + 0, + ] + assert calibration.student_loan_repayments.tolist() == [ + pytest.approx(growth, rel=1e-15), + pytest.approx(growth, rel=1e-15), + ] + + def test_every_row_is_a_level_index(): levels = TABLE.to_numpy() assert (TABLE[START_YEAR] == 1).all() diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index 92eb83479..45826756e 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -89,11 +89,13 @@ def policyengine_uk_load_time_index( each year, every variable listed in ``uprating_indices.yaml`` is multiplied by one plus the year-on-year growth parameter it is listed under. Variables the file does not list are carried forward unchanged, - and a variable's own ``uprating`` attribute only applies after the last - year the dataset is projected to, so neither appears here; past that year - this index keeps compounding the same growth parameters. Council tax and - rent, which the engine uprates by country and region, are not single - indices and are not covered. + and a variable's own ``uprating`` attribute only applies to periods with + no stored value (for a dataset column, after the last year the dataset is + projected to), so neither appears here; past that year this index keeps + compounding the same growth parameters. Council tax and rent, which the + engine uprates by country and by region and tenure, are not single + indices and are not covered, nor is the engine's yearly reassignment of + student loan plans, which zeroes the repayments of loans it writes off. """ from policyengine_uk.system import system From c4c24a261e123d39f953201718551fcb0d17fa18 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 19:10:27 -0400 Subject: [PATCH 11/21] Age the student loan fixture from the projected year Review r2 of #541: policyengine-uk carries ages unchanged at load, so ages taken from the base year put the engine's university starts at 2013 and 1983 rather than the 2012 and 1982 the comment names. Deriving them from the projected year makes the cohorts exact; 2012 is the first Plan 2 year. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/tests/test_uprating_factors_table.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/policyengine_uk_data/tests/test_uprating_factors_table.py b/policyengine_uk_data/tests/test_uprating_factors_table.py index a17f24d28..c10d44cce 100644 --- a/policyengine_uk_data/tests/test_uprating_factors_table.py +++ b/policyengine_uk_data/tests/test_uprating_factors_table.py @@ -197,8 +197,9 @@ def test_student_loan_plan_reassignment_is_outside_the_table(): "person_id": [1, 2], "person_benunit_id": [1, 2], "person_household_id": [1, 1], - # University starts in 2012 (Plan 2) and 1982 (written off). - "age": [base_year - 2012 + 18.0, base_year - 1982 + 18.0], + # Ages are carried unchanged, so in `year` these borrowers + # started university in 2012 (Plan 2) and 1982 (written off). + "age": [year - 2012 + 18.0, year - 1982 + 18.0], "student_loan_plan": ["PLAN_2", "PLAN_1"], "highest_education": ["TERTIARY", "TERTIARY"], "student_loan_repayments": [1.0, 1.0], From c65d3810ced2f1ca31cab7f6b63e82b45acb5289 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 20:04:36 -0400 Subject: [PATCH 12/21] Decide unknown-region support with a known-region control Review of #540 round 2 (Sol 6.1, REQUEST CHANGES): - model_simulates_unknown_region no longer matches error text. A failure with region UNKNOWN reads as "cannot simulate" only if the same household labelled SOUTH_EAST simulates, which is exactly when the loader's relabel helps; otherwise the control's error is raised. The household mirrors the SPI-shaped fixture policyengine-uk's own test_rent_uprating.py simulates. (ParameterNotFoundError.name is reset to None by AttributeError.__init__, so the parameter name is not available to match.) - Probe tests cover unrelated parameters ending in .UNKNOWN, unrelated parameters and schema errors (all raised), and a non-parameter failure that relabelling cures (read as "no"). - The release differential test runs only for an unmodified final release installed from a package index and imported from there: it skips pre, dev and local versions, direct-URL or editable installs, and an economic_assumptions.py that does not match the release's RECORD. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/spi.py | 42 ++++++----- policyengine_uk_data/tests/test_spi_build.py | 77 +++++++++++++++----- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index 9f661f8bb..9108d6af5 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -46,26 +46,18 @@ } -@cache -def model_simulates_unknown_region() -> bool: - """Whether the imported policyengine-uk can simulate Region.UNKNOWN. - - Releases before 2.104.5 have no rent index for it - (PolicyEngine/policyengine-uk#1985). This simulates one household rather - than reading the installed version, which need not be the imported code - (``make data-local`` puts a checkout on PYTHONPATH). - """ - from policyengine_core.errors import ParameterNotFoundError - from policyengine_uk import Microsimulation - +def _spi_shaped_household(region: str) -> UKSingleYearDataset: + # The fixture policyengine-uk's own test_rent_uprating.py simulates with + # Region.UNKNOWN, so the model keeps supporting this shape. ids = [1] - household = UKSingleYearDataset( + return UKSingleYearDataset( person=pd.DataFrame( { "person_id": ids, "person_benunit_id": ids, "person_household_id": ids, "age": [40], + "employment_income": [60_000.0], } ), benunit=pd.DataFrame({"benunit_id": ids}), @@ -73,7 +65,7 @@ def model_simulates_unknown_region() -> bool: { "household_id": ids, "household_weight": [1.0], - "region": ["UNKNOWN"], + "region": [region], "rent": [0.0], "tenure_type": ["OWNED_OUTRIGHT"], "council_tax": [0.0], @@ -81,11 +73,25 @@ def model_simulates_unknown_region() -> bool: ), fiscal_year=SPI_FISCAL_YEAR, ) + + +@cache +def model_simulates_unknown_region() -> bool: + """Whether the imported policyengine-uk can simulate Region.UNKNOWN. + + Releases before 2.104.5 have no rent index for it + (PolicyEngine/policyengine-uk#1985). This simulates one household rather + than reading the installed version, which need not be the imported code + (``make data-local`` puts a checkout on PYTHONPATH). A failure counts as + "no" only if the same household labelled SOUTH_EAST simulates; otherwise + that error is raised, since relabelling would not help. + """ + from policyengine_uk import Microsimulation + try: - Microsimulation(dataset=household) - except ParameterNotFoundError as error: - if ".UNKNOWN'" not in str(error): - raise + Microsimulation(dataset=_spi_shaped_household("UNKNOWN")) + except Exception: + Microsimulation(dataset=_spi_shaped_household("SOUTH_EAST")) return False return True diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index a6a26109a..970170f66 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -596,60 +596,101 @@ def test_create_spi_scottish_taxpayer_status_survives_h5_round_trip(tmp_path): assert loaded.household["region"].tolist() == ["UNKNOWN", "SCOTLAND", "LONDON"] -UNKNOWN_RENT_INDEX = ( +def _missing(name): + return ParameterNotFoundError(name, "2023-01-01", "rent") + + +UNKNOWN_RENT_INDEX = _missing( "gov.economic_assumptions.yoy_growth.ons.private_rental_prices.UNKNOWN" ) @pytest.mark.parametrize( - "missing, expected", [(None, True), (UNKNOWN_RENT_INDEX, False)] + "errors, expected", + [ + ({}, True), + ({"UNKNOWN": UNKNOWN_RENT_INDEX}, False), + # Any failure that relabelling to SOUTH_EAST cures. + ({"UNKNOWN": KeyError("UNKNOWN")}, False), + ], ) -def test_unknown_region_probe_reads_the_simulation(monkeypatch, missing, expected): +def test_unknown_region_probe_reads_the_simulation(monkeypatch, errors, expected): import policyengine_uk + regions = [] + def simulate(dataset): - assert dataset.household["region"].tolist() == ["UNKNOWN"] - if missing: - raise ParameterNotFoundError(missing, "2023-01-01") + (region,) = dataset.household["region"] + regions.append(region) + if region in errors: + raise errors[region] monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) assert model_simulates_unknown_region.__wrapped__() is expected + assert regions == (["UNKNOWN"] if expected else ["UNKNOWN", "SOUTH_EAST"]) -def test_unknown_region_probe_raises_other_missing_parameters(monkeypatch): +@pytest.mark.parametrize( + "error", + [ + _missing("gov.hmrc.unrelated.UNKNOWN"), + _missing("gov.hmrc.income_tax.rates.uk"), + KeyError("employment_income"), + ], +) +def test_unknown_region_probe_raises_failures_relabelling_would_not_cure( + monkeypatch, error +): + """A failure that the SOUTH_EAST household shares is raised, not read as + a missing UNKNOWN index.""" import policyengine_uk def simulate(dataset): - raise ParameterNotFoundError("gov.hmrc.income_tax.rates.uk", "2023-01-01") + raise error monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) - with pytest.raises(ParameterNotFoundError, match=r"rates\.uk'"): + with pytest.raises(type(error)) as raised: model_simulates_unknown_region.__wrapped__() + assert raised.value is error def test_unknown_region_probe_agrees_with_policyengine_uk_release(): """2.104.5 is the first policyengine-uk release with - PolicyEngine/policyengine-uk#1985. Where the imported model is the - installed release, the probe agrees with its version.""" + PolicyEngine/policyengine-uk#1985. For an unmodified final release from a + package index that is also the imported code, the probe agrees with the + release number.""" + import base64 + import hashlib from importlib.metadata import PackageNotFoundError, distribution from pathlib import Path - import policyengine_uk from packaging.version import Version + from policyengine_uk.data import economic_assumptions try: release = distribution("policyengine-uk") except PackageNotFoundError: pytest.skip("policyengine-uk is not installed as a distribution") - installed = Path(release.locate_file("policyengine_uk/__init__.py")) - if not installed.exists() or not installed.samefile(policyengine_uk.__file__): + version = Version(release.version) + if version.is_prerelease or version.is_devrelease or version.local: + pytest.skip(f"policyengine-uk {version} is not a final release") + if release.read_text("direct_url.json") is not None: + pytest.skip("policyengine-uk was not installed from a package index") + # #1985 changed this module, so it must be the installed file, unmodified. + path = "policyengine_uk/data/economic_assumptions.py" + installed = Path(release.locate_file(path)) + if not installed.exists() or not installed.samefile(economic_assumptions.__file__): pytest.skip("the imported policyengine-uk is not the installed release") - - assert model_simulates_unknown_region() == ( - Version(release.version) >= Version("2.104.5") - ) + record = {file.as_posix(): file.hash for file in release.files or []}.get(path) + if record is None: + pytest.skip("policyengine-uk's RECORD does not list the module") + digest = hashlib.new(record.mode, installed.read_bytes()).digest() + if base64.urlsafe_b64encode(digest).rstrip(b"=").decode() != record.value: + pytest.skip("the installed policyengine-uk has been modified") + + assert model_simulates_unknown_region() == (version >= Version("2.104.5")) # GORCODE, SCOT_TXP: abroad, unknown, composite, London, Scotland, abroad and From 4f86ceeeb844d6fcd76f8da7e68e5d2a8062525c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 5 Oct 2026 20:57:21 -0400 Subject: [PATCH 13/21] Scope the probe and release-test wording; pin which error the probe raises Review of #540 round 3 (Sol 6.1, APPROVE WITH CHANGES, three nits): - The probe docstring says what it establishes: that the imported model can build a simulation of an SPI-shaped household in Region.UNKNOWN. - The release differential test describes its checks as they are: a final release not installed from a direct URL, whose imported economic_assumptions.py is the installed file and matches its RECORD. - The shared-failure test gives the UNKNOWN and SOUTH_EAST households distinct errors and requires the control's, with the UNKNOWN error as its context. A mutant that re-raised the UNKNOWN error had survived. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/spi.py | 9 +++++---- policyengine_uk_data/tests/test_spi_build.py | 20 ++++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index 9108d6af5..b71eccaab 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -77,13 +77,14 @@ def _spi_shaped_household(region: str) -> UKSingleYearDataset: @cache def model_simulates_unknown_region() -> bool: - """Whether the imported policyengine-uk can simulate Region.UNKNOWN. + """Whether the imported policyengine-uk can build a simulation of an + SPI-shaped household in Region.UNKNOWN. Releases before 2.104.5 have no rent index for it - (PolicyEngine/policyengine-uk#1985). This simulates one household rather - than reading the installed version, which need not be the imported code + (PolicyEngine/policyengine-uk#1985). This builds one household rather than + reading the installed version, which need not be the imported code (``make data-local`` puts a checkout on PYTHONPATH). A failure counts as - "no" only if the same household labelled SOUTH_EAST simulates; otherwise + "no" only if the same household labelled SOUTH_EAST builds; otherwise that error is raised, since relabelling would not help. """ from policyengine_uk import Microsimulation diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index 970170f66..592c1daba 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -642,25 +642,29 @@ def simulate(dataset): def test_unknown_region_probe_raises_failures_relabelling_would_not_cure( monkeypatch, error ): - """A failure that the SOUTH_EAST household shares is raised, not read as - a missing UNKNOWN index.""" + """When the SOUTH_EAST household fails too, its error is raised, not read + as a missing UNKNOWN index.""" import policyengine_uk + original = RuntimeError("the UNKNOWN household failed") + def simulate(dataset): - raise error + (region,) = dataset.household["region"] + raise original if region == "UNKNOWN" else error monkeypatch.setattr(policyengine_uk, "Microsimulation", simulate) with pytest.raises(type(error)) as raised: model_simulates_unknown_region.__wrapped__() assert raised.value is error + assert raised.value.__context__ is original def test_unknown_region_probe_agrees_with_policyengine_uk_release(): """2.104.5 is the first policyengine-uk release with - PolicyEngine/policyengine-uk#1985. For an unmodified final release from a - package index that is also the imported code, the probe agrees with the - release number.""" + PolicyEngine/policyengine-uk#1985. For a final release not installed from + a direct URL, whose imported economic_assumptions.py is the installed file + and matches its RECORD, the probe agrees with the release number.""" import base64 import hashlib from importlib.metadata import PackageNotFoundError, distribution @@ -677,7 +681,7 @@ def test_unknown_region_probe_agrees_with_policyengine_uk_release(): if version.is_prerelease or version.is_devrelease or version.local: pytest.skip(f"policyengine-uk {version} is not a final release") if release.read_text("direct_url.json") is not None: - pytest.skip("policyengine-uk was not installed from a package index") + pytest.skip("policyengine-uk was installed from a direct URL") # #1985 changed this module, so it must be the installed file, unmodified. path = "policyengine_uk/data/economic_assumptions.py" installed = Path(release.locate_file(path)) @@ -688,7 +692,7 @@ def test_unknown_region_probe_agrees_with_policyengine_uk_release(): pytest.skip("policyengine-uk's RECORD does not list the module") digest = hashlib.new(record.mode, installed.read_bytes()).digest() if base64.urlsafe_b64encode(digest).rstrip(b"=").decode() != record.value: - pytest.skip("the installed policyengine-uk has been modified") + pytest.skip("the installed economic_assumptions.py has been modified") assert model_simulates_unknown_region() == (version >= Version("2.104.5")) From 4fbd8be43b7dc550d9f6c2871ddc7270fec38c01 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 14:40:42 -0400 Subject: [PATCH 14/21] Align lockfile project version with batch release 1.58.0 --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 5ae821352..dce9c8285 100644 --- a/uv.lock +++ b/uv.lock @@ -1433,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" }, From 31888a2d52aaf49bd860941a9d5e1ad111275d1d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 15:06:29 -0400 Subject: [PATCH 15/21] Align editable lock metadata with the 1.58.0 release The release updated pyproject.toml but left the editable package version at 1.57.4 in uv.lock, making uv lock --check fail. Update only that metadata entry; dependency versions and hashes, including UK 2.122.2 and core 3.32.13, remain unchanged. --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 5ae821352..dce9c8285 100644 --- a/uv.lock +++ b/uv.lock @@ -1433,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" }, From 2c3786f91a245cc3419638bd7027a0fa84dfdd10 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:38:57 -0400 Subject: [PATCH 16/21] Remove the 32 changelog fragments towncrier never compiled None of these files carries a towncrier type (..md), so towncrier left each one in changelog.d instead of compiling it, and several releases record "No significant changes." Every fragment's commit is in a release tagged 1.57.3 or earlier, and none of their text is in CHANGELOG.md. Renaming them with a type would list long-shipped changes under the next version, so they are removed; git history keeps their text. They also kept the old changelog check passing for any pull request, because it only counted files in changelog.d, and they stopped bump_version.py from ever reporting that no fragments were present. Co-Authored-By: Claude Opus 5.5 --- changelog.d/218.md | 1 - changelog.d/281.md | 1 - changelog.d/316.md | 1 - changelog.d/317.md | 1 - changelog.d/341.md | 1 - changelog.d/368.md | 1 - changelog.d/378.md | 1 - changelog.d/402.md | 1 - changelog.d/409.md | 1 - changelog.d/414.md | 1 - changelog.d/428.md | 2 -- changelog.d/429.md | 1 - changelog.d/431.md | 1 - changelog.d/433.md | 2 -- changelog.d/436.md | 1 - changelog.d/462.md | 1 - changelog.d/64.md | 1 - changelog.d/68.md | 1 - changelog.d/73.md | 1 - changelog.d/bus-fare-distribution-anchoring.md | 1 - changelog.d/cgt-band-donors.md | 1 - changelog.d/cgt-donor-sparsity.md | 1 - changelog.d/frs-year-semantics.md | 3 --- changelog.d/fuel-litre-calibration.md | 1 - changelog.d/fuel-uprating-household-weight.md | 1 - changelog.d/hmrc-cgt-targets.md | 1 - changelog.d/longwise-local-geography.md | 1 - changelog.d/national-age-bands.md | 1 - changelog.d/obr-efo-fallback.md | 1 - changelog.d/scottish-ct-water-charges.md | 1 - changelog.d/spi-2022-23.md | 1 - changelog.d/spi-prior-diagnostics.md | 1 - 32 files changed, 36 deletions(-) delete mode 100644 changelog.d/218.md delete mode 100644 changelog.d/281.md delete mode 100644 changelog.d/316.md delete mode 100644 changelog.d/317.md delete mode 100644 changelog.d/341.md delete mode 100644 changelog.d/368.md delete mode 100644 changelog.d/378.md delete mode 100644 changelog.d/402.md delete mode 100644 changelog.d/409.md delete mode 100644 changelog.d/414.md delete mode 100644 changelog.d/428.md delete mode 100644 changelog.d/429.md delete mode 100644 changelog.d/431.md delete mode 100644 changelog.d/433.md delete mode 100644 changelog.d/436.md delete mode 100644 changelog.d/462.md delete mode 100644 changelog.d/64.md delete mode 100644 changelog.d/68.md delete mode 100644 changelog.d/73.md delete mode 100644 changelog.d/bus-fare-distribution-anchoring.md delete mode 100644 changelog.d/cgt-band-donors.md delete mode 100644 changelog.d/cgt-donor-sparsity.md delete mode 100644 changelog.d/frs-year-semantics.md delete mode 100644 changelog.d/fuel-litre-calibration.md delete mode 100644 changelog.d/fuel-uprating-household-weight.md delete mode 100644 changelog.d/hmrc-cgt-targets.md delete mode 100644 changelog.d/longwise-local-geography.md delete mode 100644 changelog.d/national-age-bands.md delete mode 100644 changelog.d/obr-efo-fallback.md delete mode 100644 changelog.d/scottish-ct-water-charges.md delete mode 100644 changelog.d/spi-2022-23.md delete mode 100644 changelog.d/spi-prior-diagnostics.md diff --git a/changelog.d/218.md b/changelog.d/218.md deleted file mode 100644 index c5e09f604..000000000 --- a/changelog.d/218.md +++ /dev/null @@ -1 +0,0 @@ -Restrict income projection reweighting to employment and self-employment targets only, fixing ~2.5x inflation of dividend, property, and pension income (#218). diff --git a/changelog.d/281.md b/changelog.d/281.md deleted file mode 100644 index 4dce3b60a..000000000 --- a/changelog.d/281.md +++ /dev/null @@ -1 +0,0 @@ -Impute below-threshold England student loan holders into the FRS base dataset and add SLC liable-to-repay calibration targets for Plans 2 and 5. diff --git a/changelog.d/316.md b/changelog.d/316.md deleted file mode 100644 index 3addacfb4..000000000 --- a/changelog.d/316.md +++ /dev/null @@ -1 +0,0 @@ -Add Scotland and Wales private-rent country targets so constituency calibration no longer relies only on UK-wide rent anchors for those countries. diff --git a/changelog.d/317.md b/changelog.d/317.md deleted file mode 100644 index 738028e7c..000000000 --- a/changelog.d/317.md +++ /dev/null @@ -1 +0,0 @@ -Fix private rent target to use UK-wide figures and add social rent calibration target (#317). diff --git a/changelog.d/341.md b/changelog.d/341.md deleted file mode 100644 index dad119f0e..000000000 --- a/changelog.d/341.md +++ /dev/null @@ -1 +0,0 @@ -Split specific student-finance grant capacity out of the generic FRS education-grants residual and seed Disabled Students' Allowance expenses where reported grants plausibly identify DSA (#341). diff --git a/changelog.d/368.md b/changelog.d/368.md deleted file mode 100644 index 3d6318809..000000000 --- a/changelog.d/368.md +++ /dev/null @@ -1 +0,0 @@ -- Set Marriage Allowance take-up rate to 0.5 (HMRC outturn ~2.1m claimants of ~4.2m eligible couples) instead of the placeholder 1.0, so microsimulation no longer overstates Marriage Allowance cost by ~£500m/year. diff --git a/changelog.d/378.md b/changelog.d/378.md deleted file mode 100644 index 9f235903b..000000000 --- a/changelog.d/378.md +++ /dev/null @@ -1 +0,0 @@ -- Calibrate against self-employed NICs (combined OBR Class 2 + Class 4 line) and skip the inert Class 3 target so calibration diagnostics are no longer misleading (#378, partial close of #88). diff --git a/changelog.d/402.md b/changelog.d/402.md deleted file mode 100644 index 0672cba4a..000000000 --- a/changelog.d/402.md +++ /dev/null @@ -1 +0,0 @@ -- Override CPI and fixed fuel uplifts for `petrol_spending` / `diesel_spending` with a UK road-fuel volume index from HMRC clearances and OBR-implied forecast volumes, so household-level petrol and diesel litres no longer grow with consumer-price inflation. Resolves #402. diff --git a/changelog.d/409.md b/changelog.d/409.md deleted file mode 100644 index 42036bf54..000000000 --- a/changelog.d/409.md +++ /dev/null @@ -1 +0,0 @@ -- Update the UK data build pipeline to target FRS 2024-25 and flatten the UK Data Service TAB zip layout during prerequisite extraction. diff --git a/changelog.d/414.md b/changelog.d/414.md deleted file mode 100644 index 23f183731..000000000 --- a/changelog.d/414.md +++ /dev/null @@ -1 +0,0 @@ -- Set SPI trading and property allowance overrides on the concrete deduction variables, matching the PE-UK removal of the pass-through allowance variables. diff --git a/changelog.d/428.md b/changelog.d/428.md deleted file mode 100644 index ba3375978..000000000 --- a/changelog.d/428.md +++ /dev/null @@ -1,2 +0,0 @@ -- Impute household `bus_fare_spending` (COICOP 7.3.2 bus & coach fares) from the detailed LCFS codes as a new output of the consumption QRF, providing the passenger fare households pay (distinct from the ETB-based `bus_subsidy_spending`) as a building block for modelling bus fare reforms. Re-enables the `bus_subsidy_spending` smoke-test target (GBP 2.5bn) and records the `bus_fare_spending` target (GBP 3.4bn, DfT Annual Bus Statistics year ending March 2025) to enable once a dataset built with the imputation is published. Refs #427. -- Retry OBR detailed-forecast-table downloads with exponential backoff on transient HTTP errors (429/5xx) and connection failures, so an occasional OBR rate-limit no longer drops the OBR target set and reds the build. diff --git a/changelog.d/429.md b/changelog.d/429.md deleted file mode 100644 index 4ddf4c038..000000000 --- a/changelog.d/429.md +++ /dev/null @@ -1 +0,0 @@ -- Bump the locked `policyengine-uk` to `>=2.89.1` so the dataset build runs against a version that defines the `bus_fare_spending` input variable; without it the imputed bus fare column was dropped when materialising the dataset through the model and never reached the published dataset. diff --git a/changelog.d/431.md b/changelog.d/431.md deleted file mode 100644 index a8180a96f..000000000 --- a/changelog.d/431.md +++ /dev/null @@ -1 +0,0 @@ -- Calibrate `bus_fare_spending` and `bus_subsidy_spending` to DfT Annual Bus Statistics (year ending March 2025, England) totals — passenger fare receipts £3.4bn (table BUS05aii) and net government support £3.0bn (table BUS05bii), uplifted England→UK by population — via post-calibration scaling steps mirroring the rail subsidy calibration. Without anchoring, imputed bus fare inherited the broader transport-consumption over-estimate (~£10bn, ~3× too high) and bus subsidy drifted low (~£1.5bn). Adds tests asserting both bus totals match the DfT targets within 20% in the built dataset (skipped in PR CI where no dataset is built, active on the post-merge build, like test_energy_calibration). diff --git a/changelog.d/433.md b/changelog.d/433.md deleted file mode 100644 index feab253de..000000000 --- a/changelog.d/433.md +++ /dev/null @@ -1,2 +0,0 @@ -- Populate `employment_sector` (public/private, from FRS `mjobsect`) and `sic_industry_division` (SIC 2007, from FRS `sic`) Person-level variables in the FRS dataset. -- Add a national calibration target for public-sector employment (`employment_sector == PUBLIC`) against the ONS Public Sector Employment headcount. diff --git a/changelog.d/436.md b/changelog.d/436.md deleted file mode 100644 index 94147feb1..000000000 --- a/changelog.d/436.md +++ /dev/null @@ -1 +0,0 @@ -- Fix household weights (and all household-level variables) landing on the wrong households when the raw FRS household table is not ordered by `sernum` (the case from FRS 2024-25 onward), by sorting the household frame by `household_id`. Previously this scrambled the grossing weights, collapsing the modelled UK population (e.g. 18-24 from ~5.0m to ~3.0m) and skewing it old. diff --git a/changelog.d/462.md b/changelog.d/462.md deleted file mode 100644 index f72d41c5c..000000000 --- a/changelog.d/462.md +++ /dev/null @@ -1 +0,0 @@ -Sort the benunit table by benunit_id in create_frs so benunit-level variables align with the model's sorted entity order (raw 2024-25 FRS tables are no longer ordered by sernum). diff --git a/changelog.d/64.md b/changelog.d/64.md deleted file mode 100644 index 7dc7ae6a4..000000000 --- a/changelog.d/64.md +++ /dev/null @@ -1 +0,0 @@ -Use country-specific age profiles for Northern Ireland instead of UK-wide mean in local area demographic targets. diff --git a/changelog.d/68.md b/changelog.d/68.md deleted file mode 100644 index 85c2bf2fa..000000000 --- a/changelog.d/68.md +++ /dev/null @@ -1 +0,0 @@ -Average imputed earnings percentiles with national reference values to improve stability for local areas. diff --git a/changelog.d/73.md b/changelog.d/73.md deleted file mode 100644 index 80370f5d2..000000000 --- a/changelog.d/73.md +++ /dev/null @@ -1 +0,0 @@ -Assign `is_parent` from FRS adult-table membership and benefit-unit dependent-child counts (#73). diff --git a/changelog.d/bus-fare-distribution-anchoring.md b/changelog.d/bus-fare-distribution-anchoring.md deleted file mode 100644 index 4df2f3ad0..000000000 --- a/changelog.d/bus-fare-distribution-anchoring.md +++ /dev/null @@ -1 +0,0 @@ -- Anchor the distribution of bus_fare_spending to DfT BUS05ai (London/outside-London receipts split) and NTS0705a (local-bus trip rates by income quintile), preserving the national BUS05aii total, with post-build validation tests. diff --git a/changelog.d/cgt-band-donors.md b/changelog.d/cgt-band-donors.md deleted file mode 100644 index 04fafeca6..000000000 --- a/changelog.d/cgt-band-donors.md +++ /dev/null @@ -1 +0,0 @@ -- Stack donor households carrying HMRC Table 2.1a size-of-gain band mean gains, and calibrate to per-band taxpayer-count and gains targets. The spline-based capital gains imputation cannot produce gains above ~£2m (only ~1.6% of built gains came from £1m+ gains against HMRC's ~61%), and the aggregate #440 targets gave calibration no records to upweight and no constraint on the distribution's shape. Following the SPI synthetic-household precedent, 300 donors per HMRC band, entering at band-exact initial weights (published band taxpayer count / donors per band), (from £12,300 to £5m+) are stacked after the capital gains imputation, sampled with probability proportional to Advani & Summers percent_with_gains at the household's first adult's income; per-band count and amount targets from the committed HMRC size-of-gain table let calibration decide each band's weight. diff --git a/changelog.d/cgt-donor-sparsity.md b/changelog.d/cgt-donor-sparsity.md deleted file mode 100644 index d8c1e6983..000000000 --- a/changelog.d/cgt-donor-sparsity.md +++ /dev/null @@ -1 +0,0 @@ -- Reduce CGT band donors from 300 to 30 per HMRC size-of-gain band. The L0-regularised calibration pays a gate penalty per active record, so the first full build with 300 light donors per band (weight ~7 each in the top band) shrank total donor weight ~8x and left the £2m+ bands 94-97% under target, failing the built-dataset outcome tests before upload. Thirty donors per band carry the same band-exact initial totals with a tenth of the gate cost, making the per-band HMRC targets cheap for the optimiser to satisfy. diff --git a/changelog.d/frs-year-semantics.md b/changelog.d/frs-year-semantics.md deleted file mode 100644 index 1872aa188..000000000 --- a/changelog.d/frs-year-semantics.md +++ /dev/null @@ -1,3 +0,0 @@ -- Threshold FRS disability categories against the survey fiscal year's DWP rates, the same fiscal-converted parameter tree the disability flags read, and convert reported amounts back to weekly with the survey's 365.25/7 factor in both paths (uk-data#475, uk-data#476). -- Validate `create_frs`'s `year` against the FRS release folder being read, and add a `policy_year` argument (the release's calibration year in the build) so Disabled Students' Allowance expenses are seeded for the FRS 2024-25 dataset (uk-data#477, uk-data#478). -- Document that the saved base-year dataset is not uprate-invertible on the post-calibration rail, bus and road-fuel scalings (uk-data#479). diff --git a/changelog.d/fuel-litre-calibration.md b/changelog.d/fuel-litre-calibration.md deleted file mode 100644 index 09541c2fd..000000000 --- a/changelog.d/fuel-litre-calibration.md +++ /dev/null @@ -1 +0,0 @@ -- Calibrate final petrol and diesel litre proxies to HMRC/OBR road-fuel clearances, and uprate fuel-spending proxies by road-fuel litres and pump prices. diff --git a/changelog.d/fuel-uprating-household-weight.md b/changelog.d/fuel-uprating-household-weight.md deleted file mode 100644 index 03623d7f0..000000000 --- a/changelog.d/fuel-uprating-household-weight.md +++ /dev/null @@ -1 +0,0 @@ -Restore the pre-fuel-change household-weight uprating row when calibrating fuel litre proxies. diff --git a/changelog.d/hmrc-cgt-targets.md b/changelog.d/hmrc-cgt-targets.md deleted file mode 100644 index 5f1fd525f..000000000 --- a/changelog.d/hmrc-cgt-targets.md +++ /dev/null @@ -1 +0,0 @@ -- Add HMRC capital gains calibration targets (`hmrc/capital_gains_total`, `hmrc/cgt_taxpayers`) so the calibration constrains the distribution of gains, not just CGT revenue. Both apply an annual-exempt-amount gate that tracks the policy year, matching HMRC's CGT-taxpayer concept. diff --git a/changelog.d/longwise-local-geography.md b/changelog.d/longwise-local-geography.md deleted file mode 100644 index 2a986da6d..000000000 --- a/changelog.d/longwise-local-geography.md +++ /dev/null @@ -1 +0,0 @@ -- Add an OA-first long-format local geography weights artifact (`local_geography_weights.csv.gz`) so UK constituency and local-authority consumers can migrate away from dense area-by-household weight matrices. diff --git a/changelog.d/national-age-bands.md b/changelog.d/national-age-bands.md deleted file mode 100644 index 64e5998ba..000000000 --- a/changelog.d/national-age-bands.md +++ /dev/null @@ -1 +0,0 @@ -- Narrow the national gender x age calibration targets from 15-year to five-year bands. The wide bands pinned only their own totals, leaving age composition inside them unconstrained nationally, which collapsed the modelled 18-24 population to ~3.4M against ONS's ~5.4M. Also skip the superseded 15-year rows in `demographics.csv` so the two sets cannot double-count. diff --git a/changelog.d/obr-efo-fallback.md b/changelog.d/obr-efo-fallback.md deleted file mode 100644 index 200a3aa5e..000000000 --- a/changelog.d/obr-efo-fallback.md +++ /dev/null @@ -1 +0,0 @@ -- Commit the OBR EFO receipts and expenditure workbooks and fall back to them when the download fails. obr.uk serves 403 Forbidden to GitHub Actions runner IPs, and the swallowed error silently dropped 28 OBR targets, calibrating a degraded dataset that failed the reform-impact and target-registry tests (blocking the data upload). Permanent HTTP errors no longer burn the retry budget, and the committed workbooks — following the committed-not-fetched precedent of the HMRC size-of-gain table and ONS demographics — guarantee the full OBR target set builds offline. Update them together with the sources.yaml URLs on each EFO vintage change. diff --git a/changelog.d/scottish-ct-water-charges.md b/changelog.d/scottish-ct-water-charges.md deleted file mode 100644 index 1223fb625..000000000 --- a/changelog.d/scottish-ct-water-charges.md +++ /dev/null @@ -1 +0,0 @@ -Net Scottish Water and sewerage charges out of FRS-reported council tax (CTANNUAL includes them in Scotland), fixing a ~£500-per-household double count that overstated Scottish council tax by roughly 25% relative to Scottish Government receipts; the charges remain captured in water_and_sewerage_charges. diff --git a/changelog.d/spi-2022-23.md b/changelog.d/spi-2022-23.md deleted file mode 100644 index d61f1e622..000000000 --- a/changelog.d/spi-2022-23.md +++ /dev/null @@ -1 +0,0 @@ -Update SPI private prerequisites and income imputation to the 2022-23 Public Use Tape. diff --git a/changelog.d/spi-prior-diagnostics.md b/changelog.d/spi-prior-diagnostics.md deleted file mode 100644 index 51c1bd578..000000000 --- a/changelog.d/spi-prior-diagnostics.md +++ /dev/null @@ -1 +0,0 @@ -- Give zero-weight SPI synthetic households meaningful calibration prior mass, tag SPI and capital-gains synthetic rows in the enhanced FRS, and add source-weight/loss diagnostics to calibration target logs. From 99a9a34ebaa2e9deffab5b08117f126f26abb514 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:39:08 -0400 Subject: [PATCH 17/21] Require each pull request to add a typed changelog fragment The old check passed whenever changelog.d held any file, so it passed for every pull request while the untyped fragments sat on main. The new check (.github/check_changelog_fragment.py) reads the pull request's own diff, the merge commit against its first parent, and: - rejects any fragment the pull request adds or edits unless it is named changelog.d/..md with a type from pyproject.toml's towncrier config, or if it is empty; - requires at least one such fragment, added by the pull request, whenever it changes a file a release ships. Changes only to docs/, .github/, .claude/, tools/, changelog.d/, root Markdown files or .gitignore need none, so docs and CI pull requests stay out of data releases. Tests cover the rules with Hypothesis properties, differential checks that every accepted name gets the same type from towncrier's own parser and from bump_version.py, the 32 historical untyped names, and end-to-end runs on temporary git repositories. Co-Authored-By: Claude Opus 5.5 --- .github/check_changelog_fragment.py | 124 +++++++++ .github/workflows/changelog_entry.yaml | 19 +- .../tests/test_changelog_fragment_check.py | 245 ++++++++++++++++++ 3 files changed, 379 insertions(+), 9 deletions(-) create mode 100644 .github/check_changelog_fragment.py create mode 100644 policyengine_uk_data/tests/test_changelog_fragment_check.py diff --git a/.github/check_changelog_fragment.py b/.github/check_changelog_fragment.py new file mode 100644 index 000000000..99f2f5b84 --- /dev/null +++ b/.github/check_changelog_fragment.py @@ -0,0 +1,124 @@ +"""Check that a pull request adds a typed towncrier changelog fragment. + +Towncrier compiles only fragments whose names carry a type from +``[tool.towncrier]`` in pyproject.toml, and it leaves any other file in +changelog.d untouched, so that entry never reaches CHANGELOG.md. This check +accepts one name form, ``changelog.d/..md``, which towncrier and +.github/bump_version.py both read as ````. + +A pull request that changes a file a release ships must add a typed fragment +of its own. One that changes only files no release ships (docs, CI, agent +settings, offline tools, changelog fragments, root Markdown files) needs none. + +Usage: python .github/check_changelog_fragment.py +""" + +import re +import subprocess +import sys +import tomllib +from collections.abc import Callable, Iterable +from pathlib import Path + +FRAGMENT_DIR = "changelog.d/" +GITKEEP = "changelog.d/.gitkeep" +UNSHIPPED_DIRS = (".claude/", ".github/", "changelog.d/", "docs/", "tools/") +UNSHIPPED_ROOT_FILES = frozenset({".gitignore"}) + + +def fragment_types(pyproject: Path) -> list[str]: + config = tomllib.loads(pyproject.read_text())["tool"]["towncrier"] + return [fragment_type["directory"] for fragment_type in config["type"]] + + +def fragment_pattern(types: Iterable[str]) -> re.Pattern: + alternatives = "|".join(re.escape(fragment_type) for fragment_type in types) + return re.compile(rf"changelog\.d/[^/]+\.(?:{alternatives})\.md") + + +def ships(path: str) -> bool: + """Whether a release ships ``path``, so changing it needs a fragment.""" + if path.startswith(UNSHIPPED_DIRS) or path in UNSHIPPED_ROOT_FILES: + return False + return "/" in path or not path.endswith(".md") + + +def check( + changes: Iterable[tuple[str, str]], + types: list[str], + read_text: Callable[[str], str], +) -> list[str]: + """Return one message per problem in ``changes``. + + ``changes`` holds ``(status, path)`` pairs as printed by + ``git diff --name-status --no-renames``, so a rename is a deletion plus an + addition. ``read_text`` returns a path's content at the pull request's head. + """ + changes = list(changes) + pattern = fragment_pattern(types) + problems = [] + added_fragments = [] + for status, path in changes: + if not path.startswith(FRAGMENT_DIR) or path == GITKEEP or status == "D": + continue + if not pattern.fullmatch(path): + problems.append( + f"{path}: name it changelog.d/..md, where is " + f"one of: {', '.join(types)}. Towncrier ignores a fragment without " + "a type, so its entry would never reach CHANGELOG.md, and this is " + "the one form that towncrier and .github/bump_version.py both read " + "the same way." + ) + elif not read_text(path).strip(): + problems.append(f"{path} is empty. Describe the change in it.") + elif status == "A": + added_fragments.append(path) + shipped = sorted({path for _, path in changes if ships(path)}) + if shipped and not added_fragments: + listed = ", ".join(shipped[:5]) + (", ..." if len(shipped) > 5 else "") + problems.append( + f"This pull request changes files that a release ships ({listed}) " + "but adds no changelog fragment. Add one with: echo 'Description.' > " + "changelog.d/..md, where is one of: " + f"{', '.join(types)}." + ) + return problems + + +def changed_paths(base: str, head: str) -> list[tuple[str, str]]: + output = subprocess.run( + ["git", "diff", "--name-status", "--no-renames", "-z", base, head], + check=True, + capture_output=True, + text=True, + ).stdout + fields = output.split("\0")[:-1] + return list(zip(fields[::2], fields[1::2])) + + +def main(argv: list[str]) -> int: + base, head = argv + changes = changed_paths(base, head) + types = fragment_types(Path("pyproject.toml")) + problems = check(changes, types, lambda path: Path(path).read_text()) + for problem in problems: + print(f"::error::{problem}") + if problems: + return 1 + fragments = [ + path + for status, path in changes + if status == "A" and fragment_pattern(types).fullmatch(path) + ] + if fragments: + print(f"Changelog fragments added: {', '.join(fragments)}") + else: + print( + "::notice::No fragment needed: this pull request changes only files " + "that no release ships." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.github/workflows/changelog_entry.yaml b/.github/workflows/changelog_entry.yaml index 8c88b1bf8..3ed5b2281 100644 --- a/.github/workflows/changelog_entry.yaml +++ b/.github/workflows/changelog_entry.yaml @@ -15,12 +15,13 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v6 - - name: Check for changelog fragment - run: | - FRAGMENTS=$(find changelog.d -type f ! -name '.gitkeep' | wc -l) - if [ "$FRAGMENTS" -eq 0 ]; then - echo "::error::No changelog fragment found in changelog.d/" - echo "Add one with: echo 'Description.' > changelog.d/\$(git branch --show-current)..md" - echo "Types: added, changed, fixed, removed, breaking" - exit 1 - fi + with: + # The pull request's merge commit and its first parent, the tip of + # main, so the check sees exactly what merging would change. + fetch-depth: 2 + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: 3.13 + - name: Check for a typed changelog fragment + run: python .github/check_changelog_fragment.py HEAD^1 HEAD diff --git a/policyengine_uk_data/tests/test_changelog_fragment_check.py b/policyengine_uk_data/tests/test_changelog_fragment_check.py new file mode 100644 index 000000000..558a9c41b --- /dev/null +++ b/policyengine_uk_data/tests/test_changelog_fragment_check.py @@ -0,0 +1,245 @@ +"""Tests for .github/check_changelog_fragment.py, the PR changelog check.""" + +import importlib.util +import subprocess +from pathlib import Path + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +REPO = Path(__file__).resolve().parents[2] + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, REPO / ".github" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +checker = _load("check_changelog_fragment") +bump_version = _load("bump_version") +TYPES = checker.fragment_types(REPO / "pyproject.toml") +BUMP = {"breaking": "major", "added": "minor", "removed": "minor"} + +# The untyped fragments that sat on main until this check landed. Towncrier +# never compiled them, so several releases say "No significant changes." +HISTORICAL_UNTYPED = """218 281 316 317 341 368 378 402 409 414 428 429 431 433 436 +462 64 68 73 bus-fare-distribution-anchoring cgt-band-donors cgt-donor-sparsity +frs-year-semantics fuel-litre-calibration fuel-uprating-household-weight +hmrc-cgt-targets longwise-local-geography national-age-bands obr-efo-fallback +scottish-ct-water-charges spi-2022-23 spi-prior-diagnostics""".split() + +names = st.text(st.sampled_from("abc-_.0129 "), min_size=1, max_size=12) +stems = st.text(st.sampled_from("abcxyz_-019"), min_size=1, max_size=8) +types = st.sampled_from(TYPES) +shipped_paths = st.one_of( + st.sampled_from(["pyproject.toml", "uv.lock", "Makefile", ".gitattributes"]), + stems.map(lambda s: f"policyengine_uk_data/{s}.py"), + stems.map(lambda s: f"policyengine_uk_data/storage/{s}.md"), +) +unshipped_paths = st.one_of( + st.sampled_from(["README.md", "CLAUDE.md", ".gitignore", ".claude/settings.json"]), + stems.map(lambda s: f"docs/{s}.md"), + stems.map(lambda s: f".github/workflows/{s}.yaml"), + stems.map(lambda s: f"tools/{s}/build.py"), + names.map(lambda n: f"changelog.d/{n}").map(lambda p: ("D", p)), + st.sampled_from(["A", "M", "D"]).map(lambda s: (s, "changelog.d/.gitkeep")), +) + + +def _changes(paths): + return [p if isinstance(p, tuple) else ("M", p) for p in paths] + + +def _check(changes, contents=None): + contents = contents or {} + return checker.check(changes, TYPES, lambda path: contents.get(path, "Entry.")) + + +def _towncrier_parse(): + builder = pytest.importorskip("towncrier._builder") + if not hasattr(builder, "parse_newfragment_basename"): + pytest.skip("towncrier no longer exposes parse_newfragment_basename") + return builder.parse_newfragment_basename + + +def test_types_come_from_towncrier_config(): + assert TYPES == ["breaking", "added", "changed", "fixed", "removed"] + + +@pytest.mark.parametrize("stem", HISTORICAL_UNTYPED) +def test_historical_untyped_fragments_are_rejected_like_towncrier_ignores_them(stem): + path = f"changelog.d/{stem}.md" + problems = _check([("A", path), ("A", "changelog.d/x.fixed.md")]) + assert len(problems) == 1 and problems[0].startswith(f"{path}: name it") + assert _towncrier_parse()(f"{stem}.md", TYPES) == (None, None, None) + + +@pytest.mark.parametrize( + "name", + ["x.added.1.md", "x.added.md.bak", "x.added.mdx", "x.Added.md", "x.added"] + + ["sub/x.added.md", "x.feature.md", "added.md"], +) +def test_near_miss_names_are_rejected(name): + path = f"changelog.d/{name}" + problems = _check([("A", path), ("A", "changelog.d/y.fixed.md")]) + assert len(problems) == 1 and problems[0].startswith(f"{path}: name it") + + +@pytest.mark.parametrize("name", ["x.added.1.md", "x.added.md.bak"]) +def test_rejected_suffixes_would_split_towncrier_from_bump_version(tmp_path, name): + # Towncrier files these under Added, but bump_version.py sees no type and + # bumps the patch number instead of the minor one. + (tmp_path / name).write_text("Entry.") + assert _towncrier_parse()(name, TYPES)[1] == "added" + assert bump_version.infer_bump(tmp_path) == "patch" + + +@pytest.mark.parametrize( + "path, shipped", + [ + ("policyengine_uk_data/datasets/frs.py", True), + ("policyengine_uk_data/tests/test_x.py", True), + ("policyengine_uk_data/storage/BRMA_DATA_SOURCES.md", True), + ("pyproject.toml", True), + ("uv.lock", True), + ("Makefile", True), + (".gitattributes", True), + ("README.md", False), + ("CHANGELOG.md", False), + ("docs/index.md", False), + (".github/workflows/push.yaml", False), + (".github/bump_version.py", False), + (".claude/settings.json", False), + ("tools/brma_households/build.py", False), + ("changelog.d/x.fixed.md", False), + (".gitignore", False), + ], +) +def test_which_paths_a_release_ships(path, shipped): + assert checker.ships(path) is shipped + + +@given(st.lists(shipped_paths, min_size=1), st.lists(unshipped_paths), names, types) +def test_a_typed_fragment_satisfies_any_shipped_change(shipped, unshipped, name, t): + fragment = ("A", f"changelog.d/{name}.{t}.md") + assert _check(_changes(shipped + unshipped) + [fragment]) == [] + + +@given(st.lists(shipped_paths, min_size=1), st.lists(unshipped_paths), names, types) +def test_a_shipped_change_without_an_added_fragment_fails(shipped, unshipped, name, t): + # Editing a fragment already on main does not count as adding one. + edited = ("M", f"changelog.d/{name}.{t}.md") + problems = _check(_changes(shipped + unshipped) + [edited]) + assert len(problems) == 1 and "adds no changelog fragment" in problems[0] + + +@given(st.lists(unshipped_paths)) +def test_changes_only_to_unshipped_files_need_no_fragment(unshipped): + assert _check(_changes(unshipped)) == [] + + +@given(st.lists(st.one_of(shipped_paths, unshipped_paths)), names) +def test_an_untyped_fragment_is_always_rejected(others, name): + path = f"changelog.d/{name}" + if checker.fragment_pattern(TYPES).fullmatch(path): + return + problems = _check(_changes(others) + [("A", path)]) + assert any(p.startswith(f"{path}: name it") for p in problems) + + +@given(st.lists(shipped_paths, min_size=1), names, types) +def test_an_empty_fragment_is_rejected_and_does_not_count(shipped, name, t): + path = f"changelog.d/{name}.{t}.md" + problems = _check(_changes(shipped) + [("A", path)], {path: " \n"}) + assert problems[0] == f"{path} is empty. Describe the change in it." + assert "adds no changelog fragment" in problems[1] + + +@given(st.lists(st.one_of(shipped_paths, unshipped_paths)), names, types, st.data()) +def test_problems_do_not_depend_on_change_order(paths, name, t, data): + changes = _changes(paths) + [("A", f"changelog.d/{name}.{t}.md")] + shuffled = data.draw(st.permutations(changes)) + assert sorted(_check(changes)) == sorted(_check(shuffled)) + + +@given(names, types) +def test_accepted_names_are_read_the_same_way_by_towncrier(name, t): + path = f"changelog.d/{name}.{t}.md" + assert checker.fragment_pattern(TYPES).fullmatch(path) + _, category, _ = _towncrier_parse()(Path(path).name, TYPES) + assert category == t + + +@settings(max_examples=50) +@given(names, types) +def test_accepted_names_give_bump_version_the_same_type(tmp_path_factory, name, t): + directory = tmp_path_factory.mktemp("changelog.d") + (directory / f"{name}.{t}.md").write_text("Entry.") + assert bump_version.infer_bump(directory) == BUMP.get(t, "patch") + + +def _git(repo, *args): + return subprocess.run( + [ + "git", + "-c", + "user.name=t", + "-c", + "user.email=t@t", + "-c", + "commit.gpgsign=false", + *args, + ], + cwd=repo, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +@pytest.mark.parametrize( + "edits, expected", + [ + ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.fixed.md": "B."}, 0), + ({"policyengine_uk_data/x.py": "y = 2"}, 1), + ({"docs/a b.md": "Docs.", ".github/workflows/w.yaml": "on: push"}, 0), + ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.md": "B."}, 1), + # Same content, so git would report a rename unless told not to. + ( + { + "changelog.d/old.md": None, + "changelog.d/old.fixed.md": "An untyped fragment already on main.", + "policyengine_uk_data/x.py": "y = 2", + }, + 0, + ), + ], +) +def test_main_reads_the_pull_requests_own_diff(tmp_path, monkeypatch, edits, expected): + files = { + "pyproject.toml": (REPO / "pyproject.toml").read_text(), + "changelog.d/.gitkeep": "", + "changelog.d/old.md": "An untyped fragment already on main.", + "policyengine_uk_data/x.py": "y = 1", + } + for path, text in files.items(): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(text) + _git(tmp_path, "init", "-q") + _git(tmp_path, "add", ".") + _git(tmp_path, "commit", "-qm", "base") + base = _git(tmp_path, "rev-parse", "HEAD") + for path, text in edits.items(): + target = tmp_path / path + if text is None: + target.unlink() + else: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + _git(tmp_path, "add", "-A") + _git(tmp_path, "commit", "-qm", "change") + monkeypatch.chdir(tmp_path) + assert checker.main([base, "HEAD"]) == expected From 943101d6e8994344f9661254e8b110d2126fde94 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:39:38 -0400 Subject: [PATCH 18/21] Document the fragment rule and add this change's own fragment CONTRIBUTING.md gains a "Changelog fragments" section: the name form, what each type does to the version, which paths need no fragment, and that merging a fragment releases data. This pull request changes the package's tests, so under its own rule it adds a typed fragment. Merging it is therefore a data release. Co-Authored-By: Claude Opus 5.5 --- .github/CONTRIBUTING.md | 7 +++++++ changelog.d/typed-changelog-fragments.fixed.md | 1 + 2 files changed, 8 insertions(+) create mode 100644 changelog.d/typed-changelog-fragments.fixed.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 73a49779f..4ba362add 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -39,6 +39,13 @@ The enhanced FRS dataset is licensed under strict UK Data Service terms. Violati If your change is a non-bugfix update to a cloud-hosted dataset (FRS, enhanced FRS), bump both the filename and URL in the class definition and in `storage/upload_completed_datasets.py`. That lets us store historical dataset versions separately and reproducibly. +## Changelog fragments + +- A pull request that changes a file a release ships adds its own fragment, `changelog.d/..md`. The type sets the version bump: `breaking` is major, `added` and `removed` are minor, and `changed` and `fixed` are patch. +- Towncrier ignores a fragment without a type, so the `Changelog entry` check rejects any other name. It also fails a pull request that changes shipped files without adding a fragment. +- A pull request that changes only `docs/`, `.github/`, `.claude/`, `tools/`, `changelog.d/`, root Markdown files or `.gitignore` needs no fragment. +- Merging a fragment into `main` releases data: the versioning workflow bumps the version, then the Push workflow builds, tests, uploads and tags it. + ## Repo-specific anti-patterns - **Don't** hardcode dataset years in variable transforms; use `dataset.time_period` and the uprating pipeline. diff --git a/changelog.d/typed-changelog-fragments.fixed.md b/changelog.d/typed-changelog-fragments.fixed.md new file mode 100644 index 000000000..1b3f41855 --- /dev/null +++ b/changelog.d/typed-changelog-fragments.fixed.md @@ -0,0 +1 @@ +Remove 32 changelog fragments that had no type, so towncrier never compiled them. Their changes shipped in 1.57.3 or earlier, but their entries never reached this changelog. The changelog check now rejects a fragment without a type and requires one from each pull request that changes a file a release ships. From bb41c3ca498fcb56177dbd2a8553f94ae28570fe Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:40:43 -0400 Subject: [PATCH 19/21] Point the fragment at #555, which lists the removed fragments Co-Authored-By: Claude Opus 5.5 --- changelog.d/typed-changelog-fragments.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/typed-changelog-fragments.fixed.md b/changelog.d/typed-changelog-fragments.fixed.md index 1b3f41855..59a7cb23a 100644 --- a/changelog.d/typed-changelog-fragments.fixed.md +++ b/changelog.d/typed-changelog-fragments.fixed.md @@ -1 +1 @@ -Remove 32 changelog fragments that had no type, so towncrier never compiled them. Their changes shipped in 1.57.3 or earlier, but their entries never reached this changelog. The changelog check now rejects a fragment without a type and requires one from each pull request that changes a file a release ships. +Remove 32 changelog fragments that had no type, so towncrier never compiled them. Their changes shipped in 1.57.3 or earlier, but their entries never reached this changelog; #555 lists them. The changelog check now rejects a fragment without a type and requires one from each pull request that changes a file a release ships. From 8808d47dc33a3d89ccccbf81fbba03cbb648a038 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 19:54:08 -0400 Subject: [PATCH 20/21] Reject fragment names that stop towncrier, and read paths as bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of bb41c3ca found two names the check accepted that stop versioning, so the release never starts: - 1.fixed.md with 01.fixed.md: towncrier drops a number's leading zeros and stops with "multiple files for 1.fixed". The check now computes the entry towncrier files each fragment under, across every fragment in changelog.d at the pull request's head, and rejects duplicates. - a name such as "².fixed.md": towncrier's isdigit() passes but int() raises. Names are now limited to ASCII letters, digits, ".", "_" and "-", starting with a letter or digit, which also rules out the whitespace towncrier strips ("foo .fixed.md" against "foo.fixed.md"). Also from the review: - git's path output is read as bytes and decoded with os.fsdecode, so a carriage return or a non-UTF-8 byte in a file name no longer changes or breaks the path. A fragment that is not UTF-8 text is rejected. - The exempt paths are described as a release-policy choice: a change under .github/ can still change how the next release is built. Tests add towncrier differentials over arbitrary Unicode names and over collisions, a real "towncrier build --draft" run, and the repo's Hypothesis settings convention (no deadline, too_slow suppressed). Co-Authored-By: Claude Opus 5.5 --- .github/CONTRIBUTING.md | 6 +- .github/check_changelog_fragment.py | 116 +++++++---- .../typed-changelog-fragments.fixed.md | 2 +- .../tests/test_changelog_fragment_check.py | 188 +++++++++++++----- 4 files changed, 216 insertions(+), 96 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4ba362add..912881ac5 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -41,9 +41,9 @@ If your change is a non-bugfix update to a cloud-hosted dataset (FRS, enhanced F ## Changelog fragments -- A pull request that changes a file a release ships adds its own fragment, `changelog.d/..md`. The type sets the version bump: `breaking` is major, `added` and `removed` are minor, and `changed` and `fixed` are patch. -- Towncrier ignores a fragment without a type, so the `Changelog entry` check rejects any other name. It also fails a pull request that changes shipped files without adding a fragment. -- A pull request that changes only `docs/`, `.github/`, `.claude/`, `tools/`, `changelog.d/`, root Markdown files or `.gitignore` needs no fragment. +- A pull request adds its own fragment, `changelog.d/..md`, unless it changes only the exempt paths below. `` uses ASCII letters, digits, `.`, `_` and `-`. The type sets the version bump: `breaking` is major, `added` and `removed` are minor, and `changed` and `fixed` are patch. +- Towncrier ignores a fragment without a type, so the `Changelog entry` check rejects any other name. It also rejects two fragments that towncrier would file as the same entry, such as `1.fixed.md` and `01.fixed.md`. +- Exempt paths: `docs/`, `.github/`, `.claude/`, `tools/`, `changelog.d/`, root Markdown files and `.gitignore`. This is a policy choice, so that docs and CI changes can merge without a data release; a change under `.github/` can still change how the next release is built. - Merging a fragment into `main` releases data: the versioning workflow bumps the version, then the Push workflow builds, tests, uploads and tags it. ## Repo-specific anti-patterns diff --git a/.github/check_changelog_fragment.py b/.github/check_changelog_fragment.py index 99f2f5b84..132056002 100644 --- a/.github/check_changelog_fragment.py +++ b/.github/check_changelog_fragment.py @@ -1,18 +1,24 @@ -"""Check that a pull request adds a typed towncrier changelog fragment. +"""Check the towncrier changelog fragments a pull request would merge. Towncrier compiles only fragments whose names carry a type from -``[tool.towncrier]`` in pyproject.toml, and it leaves any other file in -changelog.d untouched, so that entry never reaches CHANGELOG.md. This check -accepts one name form, ``changelog.d/..md``, which towncrier and -.github/bump_version.py both read as ````. - -A pull request that changes a file a release ships must add a typed fragment -of its own. One that changes only files no release ships (docs, CI, agent -settings, offline tools, changelog fragments, root Markdown files) needs none. +``[tool.towncrier]`` in pyproject.toml, and leaves any other file in +changelog.d, so that entry never reaches CHANGELOG.md. This check accepts one +name form, ``changelog.d/..md`` with an ASCII ````, which +towncrier and .github/bump_version.py both read as ````. It also rejects +two fragments that towncrier would file as the same entry, because towncrier +then stops versioning with an error. + +A pull request that changes a file outside the exempt paths must add a typed +fragment of its own. The exempt paths are docs, CI and release tooling +(.github/), agent settings, offline tools, changelog fragments, root Markdown +files and .gitignore. That is a release-policy choice, not a guarantee: a +change under .github/ can still change how the next release is built. +Exempting these paths lets docs and CI changes merge without a data release. Usage: python .github/check_changelog_fragment.py """ +import os import re import subprocess import sys @@ -22,8 +28,8 @@ FRAGMENT_DIR = "changelog.d/" GITKEEP = "changelog.d/.gitkeep" -UNSHIPPED_DIRS = (".claude/", ".github/", "changelog.d/", "docs/", "tools/") -UNSHIPPED_ROOT_FILES = frozenset({".gitignore"}) +EXEMPT_DIRS = (".claude/", ".github/", "changelog.d/", "docs/", "tools/") +EXEMPT_ROOT_FILES = frozenset({".gitignore"}) def fragment_types(pyproject: Path) -> list[str]: @@ -33,12 +39,19 @@ def fragment_types(pyproject: Path) -> list[str]: def fragment_pattern(types: Iterable[str]) -> re.Pattern: alternatives = "|".join(re.escape(fragment_type) for fragment_type in types) - return re.compile(rf"changelog\.d/[^/]+\.(?:{alternatives})\.md") + return re.compile( + rf"changelog\.d/(?P[A-Za-z0-9][A-Za-z0-9._-]*)\.(?P{alternatives})\.md" + ) + + +def towncrier_entry(match: re.Match) -> tuple[str, str]: + """The entry towncrier files a fragment under: it drops a number's leading zeros.""" + name = match["name"] + return (str(int(name)) if name.isdigit() else name, match["type"]) -def ships(path: str) -> bool: - """Whether a release ships ``path``, so changing it needs a fragment.""" - if path.startswith(UNSHIPPED_DIRS) or path in UNSHIPPED_ROOT_FILES: +def needs_fragment(path: str) -> bool: + if path.startswith(EXEMPT_DIRS) or path in EXEMPT_ROOT_FILES: return False return "/" in path or not path.endswith(".md") @@ -46,13 +59,15 @@ def ships(path: str) -> bool: def check( changes: Iterable[tuple[str, str]], types: list[str], - read_text: Callable[[str], str], + read_bytes: Callable[[str], bytes], + fragments_at_head: Iterable[str], ) -> list[str]: - """Return one message per problem in ``changes``. + """Return one message per problem. ``changes`` holds ``(status, path)`` pairs as printed by ``git diff --name-status --no-renames``, so a rename is a deletion plus an - addition. ``read_text`` returns a path's content at the pull request's head. + addition. ``read_bytes`` returns a path's content at the pull request's + head, and ``fragments_at_head`` lists every file in changelog.d there. """ changes = list(changes) pattern = fragment_pattern(types) @@ -63,21 +78,39 @@ def check( continue if not pattern.fullmatch(path): problems.append( - f"{path}: name it changelog.d/..md, where is " - f"one of: {', '.join(types)}. Towncrier ignores a fragment without " - "a type, so its entry would never reach CHANGELOG.md, and this is " - "the one form that towncrier and .github/bump_version.py both read " + f"{path}: name it changelog.d/..md, where uses " + "only ASCII letters, digits, '.', '_' and '-', and is one " + f"of: {', '.join(types)}. Towncrier ignores a fragment without a " + "type, so its entry would never reach CHANGELOG.md, and this is " + "the form that towncrier and .github/bump_version.py both read " "the same way." ) - elif not read_text(path).strip(): + continue + try: + text = read_bytes(path).decode("utf-8") + except UnicodeDecodeError: + problems.append(f"{path} is not UTF-8 text.") + continue + if not text.strip(): problems.append(f"{path} is empty. Describe the change in it.") elif status == "A": added_fragments.append(path) - shipped = sorted({path for _, path in changes if ships(path)}) - if shipped and not added_fragments: - listed = ", ".join(shipped[:5]) + (", ..." if len(shipped) > 5 else "") + entries = {} + for path in sorted(fragments_at_head): + if match := pattern.fullmatch(path): + entries.setdefault(towncrier_entry(match), []).append(path) + for (name, fragment_type), paths in sorted(entries.items()): + if len(paths) > 1: + problems.append( + f"{' and '.join(paths)}: towncrier files each as " + f"{name}.{fragment_type} and stops versioning with an error. " + "Rename all but one." + ) + needing = sorted({path for _, path in changes if needs_fragment(path)}) + if needing and not added_fragments: + listed = ", ".join(needing[:5]) + (", ..." if len(needing) > 5 else "") problems.append( - f"This pull request changes files that a release ships ({listed}) " + f"This pull request changes files outside the exempt paths ({listed}) " "but adds no changelog fragment. Add one with: echo 'Description.' > " "changelog.d/..md, where is one of: " f"{', '.join(types)}." @@ -90,32 +123,35 @@ def changed_paths(base: str, head: str) -> list[tuple[str, str]]: ["git", "diff", "--name-status", "--no-renames", "-z", base, head], check=True, capture_output=True, - text=True, ).stdout - fields = output.split("\0")[:-1] - return list(zip(fields[::2], fields[1::2])) + fields = output.split(b"\0")[:-1] + return [ + (status.decode(), os.fsdecode(path)) + for status, path in zip(fields[::2], fields[1::2]) + ] def main(argv: list[str]) -> int: base, head = argv changes = changed_paths(base, head) types = fragment_types(Path("pyproject.toml")) - problems = check(changes, types, lambda path: Path(path).read_text()) + fragments_at_head = [ + f"{FRAGMENT_DIR}{path.name}" + for path in Path(FRAGMENT_DIR).iterdir() + if path.is_file() + ] + problems = check( + changes, types, lambda path: Path(path).read_bytes(), fragments_at_head + ) for problem in problems: print(f"::error::{problem}") if problems: return 1 - fragments = [ - path - for status, path in changes - if status == "A" and fragment_pattern(types).fullmatch(path) - ] - if fragments: - print(f"Changelog fragments added: {', '.join(fragments)}") + if any(needs_fragment(path) for _, path in changes): + print("Changelog fragment check passed.") else: print( - "::notice::No fragment needed: this pull request changes only files " - "that no release ships." + "::notice::No fragment needed: this pull request changes only exempt paths." ) return 0 diff --git a/changelog.d/typed-changelog-fragments.fixed.md b/changelog.d/typed-changelog-fragments.fixed.md index 59a7cb23a..39ba2afb3 100644 --- a/changelog.d/typed-changelog-fragments.fixed.md +++ b/changelog.d/typed-changelog-fragments.fixed.md @@ -1 +1 @@ -Remove 32 changelog fragments that had no type, so towncrier never compiled them. Their changes shipped in 1.57.3 or earlier, but their entries never reached this changelog; #555 lists them. The changelog check now rejects a fragment without a type and requires one from each pull request that changes a file a release ships. +Remove 32 changelog fragments that had no type, so towncrier never compiled them. Their changes shipped in 1.57.3 or earlier, but their entries never reached this changelog; #555 lists them. The changelog check now rejects a fragment without a type, and requires a typed fragment from each pull request unless it changes only docs, CI and the other exempt paths in CONTRIBUTING.md. diff --git a/policyengine_uk_data/tests/test_changelog_fragment_check.py b/policyengine_uk_data/tests/test_changelog_fragment_check.py index 558a9c41b..1b5039fe4 100644 --- a/policyengine_uk_data/tests/test_changelog_fragment_check.py +++ b/policyengine_uk_data/tests/test_changelog_fragment_check.py @@ -1,11 +1,13 @@ """Tests for .github/check_changelog_fragment.py, the PR changelog check.""" import importlib.util +import os import subprocess +import sys from pathlib import Path import pytest -from hypothesis import given, settings +from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st REPO = Path(__file__).resolve().parents[2] @@ -21,7 +23,9 @@ def _load(name: str): checker = _load("check_changelog_fragment") bump_version = _load("bump_version") TYPES = checker.fragment_types(REPO / "pyproject.toml") +PATTERN = checker.fragment_pattern(TYPES) BUMP = {"breaking": "major", "added": "minor", "removed": "minor"} +relaxed = settings(deadline=None, suppress_health_check=[HealthCheck.too_slow]) # The untyped fragments that sat on main until this check landed. Towncrier # never compiled them, so several releases say "No significant changes." @@ -31,20 +35,25 @@ def _load(name: str): hmrc-cgt-targets longwise-local-geography national-age-bands obr-efo-fallback scottish-ct-water-charges spi-2022-23 spi-prior-diagnostics""".split() -names = st.text(st.sampled_from("abc-_.0129 "), min_size=1, max_size=12) -stems = st.text(st.sampled_from("abcxyz_-019"), min_size=1, max_size=8) +valid_names = st.from_regex(r"[A-Za-z0-9][A-Za-z0-9._-]{0,10}", fullmatch=True) +any_names = st.text( + st.characters(exclude_categories=("Cs",), exclude_characters="/\0"), + min_size=1, + max_size=12, +) types = st.sampled_from(TYPES) -shipped_paths = st.one_of( +stems = st.text(st.sampled_from("abcxyz_-019"), min_size=1, max_size=8) +needing_paths = st.one_of( st.sampled_from(["pyproject.toml", "uv.lock", "Makefile", ".gitattributes"]), stems.map(lambda s: f"policyengine_uk_data/{s}.py"), stems.map(lambda s: f"policyengine_uk_data/storage/{s}.md"), ) -unshipped_paths = st.one_of( +exempt_paths = st.one_of( st.sampled_from(["README.md", "CLAUDE.md", ".gitignore", ".claude/settings.json"]), stems.map(lambda s: f"docs/{s}.md"), stems.map(lambda s: f".github/workflows/{s}.yaml"), stems.map(lambda s: f"tools/{s}/build.py"), - names.map(lambda n: f"changelog.d/{n}").map(lambda p: ("D", p)), + any_names.map(lambda n: ("D", f"changelog.d/{n}")), st.sampled_from(["A", "M", "D"]).map(lambda s: (s, "changelog.d/.gitkeep")), ) @@ -53,9 +62,13 @@ def _changes(paths): return [p if isinstance(p, tuple) else ("M", p) for p in paths] -def _check(changes, contents=None): +def _check(changes, contents=None, at_head=None): contents = contents or {} - return checker.check(changes, TYPES, lambda path: contents.get(path, "Entry.")) + if at_head is None: + at_head = [p for s, p in changes if s != "D" and p.startswith("changelog.d/")] + return checker.check( + changes, TYPES, lambda path: contents.get(path, b"Entry."), at_head + ) def _towncrier_parse(): @@ -80,7 +93,8 @@ def test_historical_untyped_fragments_are_rejected_like_towncrier_ignores_them(s @pytest.mark.parametrize( "name", ["x.added.1.md", "x.added.md.bak", "x.added.mdx", "x.Added.md", "x.added"] - + ["sub/x.added.md", "x.feature.md", "added.md"], + + ["sub/x.added.md", "x.feature.md", "added.md", "².added.md", "x .added.md"] + + [".x.added.md", "-x.added.md", "x\r.added.md", "é.added.md"], ) def test_near_miss_names_are_rejected(name): path = f"changelog.d/{name}" @@ -98,7 +112,7 @@ def test_rejected_suffixes_would_split_towncrier_from_bump_version(tmp_path, nam @pytest.mark.parametrize( - "path, shipped", + "path, needed", [ ("policyengine_uk_data/datasets/frs.py", True), ("policyengine_uk_data/tests/test_x.py", True), @@ -118,81 +132,142 @@ def test_rejected_suffixes_would_split_towncrier_from_bump_version(tmp_path, nam (".gitignore", False), ], ) -def test_which_paths_a_release_ships(path, shipped): - assert checker.ships(path) is shipped +def test_which_paths_need_a_fragment(path, needed): + assert checker.needs_fragment(path) is needed -@given(st.lists(shipped_paths, min_size=1), st.lists(unshipped_paths), names, types) -def test_a_typed_fragment_satisfies_any_shipped_change(shipped, unshipped, name, t): +@relaxed +@given(st.lists(needing_paths, min_size=1), st.lists(exempt_paths), valid_names, types) +def test_a_typed_fragment_satisfies_any_change(needing, exempt, name, t): fragment = ("A", f"changelog.d/{name}.{t}.md") - assert _check(_changes(shipped + unshipped) + [fragment]) == [] + assert _check(_changes(needing + exempt) + [fragment]) == [] -@given(st.lists(shipped_paths, min_size=1), st.lists(unshipped_paths), names, types) -def test_a_shipped_change_without_an_added_fragment_fails(shipped, unshipped, name, t): +@relaxed +@given(st.lists(needing_paths, min_size=1), st.lists(exempt_paths), valid_names, types) +def test_a_change_needing_a_fragment_fails_without_an_added_one(needing, exempt, n, t): # Editing a fragment already on main does not count as adding one. - edited = ("M", f"changelog.d/{name}.{t}.md") - problems = _check(_changes(shipped + unshipped) + [edited]) + edited = ("M", f"changelog.d/{n}.{t}.md") + problems = _check(_changes(needing + exempt) + [edited]) assert len(problems) == 1 and "adds no changelog fragment" in problems[0] -@given(st.lists(unshipped_paths)) -def test_changes_only_to_unshipped_files_need_no_fragment(unshipped): - assert _check(_changes(unshipped)) == [] +@relaxed +@given(st.lists(exempt_paths)) +def test_changes_only_to_exempt_paths_need_no_fragment(exempt): + assert _check(_changes(exempt)) == [] -@given(st.lists(st.one_of(shipped_paths, unshipped_paths)), names) -def test_an_untyped_fragment_is_always_rejected(others, name): +@relaxed +@given(st.lists(st.one_of(needing_paths, exempt_paths)), any_names) +def test_a_badly_named_fragment_is_always_rejected(others, name): path = f"changelog.d/{name}" - if checker.fragment_pattern(TYPES).fullmatch(path): + if PATTERN.fullmatch(path): return problems = _check(_changes(others) + [("A", path)]) assert any(p.startswith(f"{path}: name it") for p in problems) -@given(st.lists(shipped_paths, min_size=1), names, types) -def test_an_empty_fragment_is_rejected_and_does_not_count(shipped, name, t): +@relaxed +@given(st.lists(needing_paths, min_size=1), valid_names, types) +def test_an_empty_fragment_is_rejected_and_does_not_count(needing, name, t): path = f"changelog.d/{name}.{t}.md" - problems = _check(_changes(shipped) + [("A", path)], {path: " \n"}) + problems = _check(_changes(needing) + [("A", path)], {path: b" \n"}) assert problems[0] == f"{path} is empty. Describe the change in it." assert "adds no changelog fragment" in problems[1] -@given(st.lists(st.one_of(shipped_paths, unshipped_paths)), names, types, st.data()) +def test_a_fragment_that_is_not_utf8_is_rejected(): + path = "changelog.d/x.fixed.md" + assert _check([("A", path)], {path: b"\xff"}) == [f"{path} is not UTF-8 text."] + + +@relaxed +@given(st.lists(st.one_of(needing_paths, exempt_paths)), valid_names, types, st.data()) def test_problems_do_not_depend_on_change_order(paths, name, t, data): changes = _changes(paths) + [("A", f"changelog.d/{name}.{t}.md")] shuffled = data.draw(st.permutations(changes)) assert sorted(_check(changes)) == sorted(_check(shuffled)) -@given(names, types) -def test_accepted_names_are_read_the_same_way_by_towncrier(name, t): - path = f"changelog.d/{name}.{t}.md" - assert checker.fragment_pattern(TYPES).fullmatch(path) - _, category, _ = _towncrier_parse()(Path(path).name, TYPES) - assert category == t +@relaxed +@given(st.one_of(valid_names, any_names), types) +def test_towncrier_reads_every_accepted_name_as_the_same_entry(name, t): + basename = f"{name}.{t}.md" + if match := PATTERN.fullmatch(f"changelog.d/{basename}"): + issue, category, counter = _towncrier_parse()(basename, TYPES) + assert (issue, category, counter) == (*checker.towncrier_entry(match), 0) -@settings(max_examples=50) -@given(names, types) -def test_accepted_names_give_bump_version_the_same_type(tmp_path_factory, name, t): - directory = tmp_path_factory.mktemp("changelog.d") - (directory / f"{name}.{t}.md").write_text("Entry.") - assert bump_version.infer_bump(directory) == BUMP.get(t, "patch") +@relaxed +@given(valid_names, valid_names, types) +def test_a_collision_is_reported_exactly_when_towncrier_would_collide(a, b, t): + paths = [f"changelog.d/{a}.{t}.md", f"changelog.d/{b}.{t}.md"] + if paths[0] == paths[1]: + return + parse = _towncrier_parse() + collide = parse(Path(paths[0]).name, TYPES) == parse(Path(paths[1]).name, TYPES) + problems = _check([("A", path) for path in paths]) + assert bool(problems) is collide + assert all("stops versioning" in p for p in problems) + + +def test_a_collision_with_a_fragment_already_on_main_is_reported(): + problems = _check( + [("A", "changelog.d/01.fixed.md")], + at_head=["changelog.d/1.fixed.md", "changelog.d/01.fixed.md"], + ) + assert problems == [ + "changelog.d/01.fixed.md and changelog.d/1.fixed.md: towncrier files " + "each as 1.fixed and stops versioning with an error. Rename all but one." + ] + + +@pytest.mark.parametrize( + "names, collide", + [ + (["1.fixed.md", "01.fixed.md"], True), + (["0.added.md", "000.added.md"], True), + (["1.fixed.md", "1.0.fixed.md"], False), + (["foo.fixed.md", "foo.added.md"], False), + ], +) +def test_towncrier_build_fails_exactly_when_the_check_reports_a_collision( + tmp_path, names, collide +): + pytest.importorskip("towncrier") + (tmp_path / "pyproject.toml").write_text((REPO / "pyproject.toml").read_text()) + (tmp_path / "changelog.d").mkdir() + for name in names: + (tmp_path / "changelog.d" / name).write_text(f"Entry {name}.") + build = subprocess.run( + [sys.executable, "-m", "towncrier", "build", "--draft"] + + ["--name", "x", "--version", "0.0.0", "--dir", str(tmp_path)], + capture_output=True, + text=True, + ) + assert (build.returncode != 0) is collide + assert bool(_check([("A", f"changelog.d/{name}") for name in names])) is collide + + +def test_changed_paths_keeps_raw_path_bytes(monkeypatch): + output = b"M\0docs/a\rb.md\0A\0changelog.d/\xff.fixed.md\0D\0changelog.d/x.md\0" + monkeypatch.setattr( + checker.subprocess, + "run", + lambda *args, **kwargs: subprocess.CompletedProcess(args, 0, stdout=output), + ) + assert checker.changed_paths("a", "b") == [ + ("M", "docs/a\rb.md"), + ("A", os.fsdecode(b"changelog.d/\xff.fixed.md")), + ("D", "changelog.d/x.md"), + ] def _git(repo, *args): + identity = ["-c", "user.name=t", "-c", "user.email=t@t"] return subprocess.run( - [ - "git", - "-c", - "user.name=t", - "-c", - "user.email=t@t", - "-c", - "commit.gpgsign=false", - *args, - ], + ["git", *identity, "-c", "commit.gpgsign=false", *args], cwd=repo, check=True, capture_output=True, @@ -205,8 +280,9 @@ def _git(repo, *args): [ ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.fixed.md": "B."}, 0), ({"policyengine_uk_data/x.py": "y = 2"}, 1), - ({"docs/a b.md": "Docs.", ".github/workflows/w.yaml": "on: push"}, 0), + ({"docs/a b.md": "Docs.", "docs/c\rd.md": "CR.", ".github/w.yaml": "x"}, 0), ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.md": "B."}, 1), + ({"changelog.d/1.fixed.md": "A.", "changelog.d/01.fixed.md": "B."}, 1), # Same content, so git would report a rename unless told not to. ( { @@ -243,3 +319,11 @@ def test_main_reads_the_pull_requests_own_diff(tmp_path, monkeypatch, edits, exp _git(tmp_path, "commit", "-qm", "change") monkeypatch.chdir(tmp_path) assert checker.main([base, "HEAD"]) == expected + + +@settings(relaxed, max_examples=50) +@given(valid_names, types) +def test_accepted_names_give_bump_version_the_same_type(tmp_path_factory, name, t): + directory = tmp_path_factory.mktemp("changelog.d") + (directory / f"{name}.{t}.md").write_text("Entry.") + assert bump_version.infer_bump(directory) == BUMP.get(t, "patch") From 6af935fb26654cc6b0da5fcb6b31832a347c6dc6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 20:26:12 -0400 Subject: [PATCH 21/21] Handle a removed changelog.d, and exempt .gitkeep in the name property Second review of 8808d47d approved it with two code notes: - A pull request that deletes every file in changelog.d leaves no such directory in the checkout, and the check raised FileNotFoundError instead of passing. It now treats a missing directory as empty. - The property "a badly named fragment is always rejected" could draw the name .gitkeep, which the check deliberately skips, and fail. The test now skips it and pins that case as an explicit example. Co-Authored-By: Claude Opus 5.5 --- .github/check_changelog_fragment.py | 5 ++++- .../tests/test_changelog_fragment_check.py | 9 +++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/check_changelog_fragment.py b/.github/check_changelog_fragment.py index 132056002..37d536cc3 100644 --- a/.github/check_changelog_fragment.py +++ b/.github/check_changelog_fragment.py @@ -135,9 +135,12 @@ def main(argv: list[str]) -> int: base, head = argv changes = changed_paths(base, head) types = fragment_types(Path("pyproject.toml")) + # Git keeps no empty directories, so changelog.d is absent once a pull + # request deletes every file in it. + fragment_dir = Path(FRAGMENT_DIR) fragments_at_head = [ f"{FRAGMENT_DIR}{path.name}" - for path in Path(FRAGMENT_DIR).iterdir() + for path in (fragment_dir.iterdir() if fragment_dir.is_dir() else []) if path.is_file() ] problems = check( diff --git a/policyengine_uk_data/tests/test_changelog_fragment_check.py b/policyengine_uk_data/tests/test_changelog_fragment_check.py index 1b5039fe4..82ac858a3 100644 --- a/policyengine_uk_data/tests/test_changelog_fragment_check.py +++ b/policyengine_uk_data/tests/test_changelog_fragment_check.py @@ -7,7 +7,7 @@ from pathlib import Path import pytest -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st REPO = Path(__file__).resolve().parents[2] @@ -160,9 +160,10 @@ def test_changes_only_to_exempt_paths_need_no_fragment(exempt): @relaxed @given(st.lists(st.one_of(needing_paths, exempt_paths)), any_names) +@example(others=[], name=".gitkeep") def test_a_badly_named_fragment_is_always_rejected(others, name): path = f"changelog.d/{name}" - if PATTERN.fullmatch(path): + if PATTERN.fullmatch(path) or path == checker.GITKEEP: return problems = _check(_changes(others) + [("A", path)]) assert any(p.startswith(f"{path}: name it") for p in problems) @@ -283,6 +284,8 @@ def _git(repo, *args): ({"docs/a b.md": "Docs.", "docs/c\rd.md": "CR.", ".github/w.yaml": "x"}, 0), ({"policyengine_uk_data/x.py": "y = 2", "changelog.d/b.md": "B."}, 1), ({"changelog.d/1.fixed.md": "A.", "changelog.d/01.fixed.md": "B."}, 1), + # Deleting every file removes changelog.d itself, as in a real checkout. + ({"changelog.d/old.md": None, "changelog.d/.gitkeep": None}, 0), # Same content, so git would report a rename unless told not to. ( { @@ -312,6 +315,8 @@ def test_main_reads_the_pull_requests_own_diff(tmp_path, monkeypatch, edits, exp target = tmp_path / path if text is None: target.unlink() + if not any(target.parent.iterdir()): + target.parent.rmdir() else: target.parent.mkdir(parents=True, exist_ok=True) target.write_text(text)