From 0b45992696e29137fb986ab0957db1621eb717fa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Thu, 1 Oct 2026 15:38:59 -0400 Subject: [PATCH 1/5] Carry the FRS benefit-unit capital into pension_credit_reported_capital TOTCAPB3 (DWP's derived benefit-unit total of savings and investments) for every FRS benefit unit; -1 where missing or negative. Pension Credit counts the claimant's and partner's capital (SPCA 2002 s.5); the WAS household imputation gives FRS Pension Credit reporters a weighted median of 74,300 pounds against the survey's own 300 for those the model then finds not entitled. Co-Authored-By: Claude Opus 5.5 --- .../frs-pc-reported-capital.changed.md | 1 + policyengine_uk_data/datasets/frs.py | 31 +++++++++++++ .../test_pension_credit_reported_capital.py | 44 +++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100644 changelog.d/frs-pc-reported-capital.changed.md create mode 100644 policyengine_uk_data/tests/test_pension_credit_reported_capital.py diff --git a/changelog.d/frs-pc-reported-capital.changed.md b/changelog.d/frs-pc-reported-capital.changed.md new file mode 100644 index 00000000..7986dc73 --- /dev/null +++ b/changelog.d/frs-pc-reported-capital.changed.md @@ -0,0 +1 @@ +Carry each benefit unit's FRS total capital (TOTCAPB3) into policyengine-uk's `pension_credit_reported_capital`, so Pension Credit's capital test uses the survey's own benefit-unit capital instead of imputed household wealth. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index e440d486..5cdb760c 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -570,6 +570,30 @@ def validate_frs_survey_year(raw_frs_folder, year: int) -> None: ) +def derive_pension_credit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: + """Each benefit unit's capital as the FRS records it, for Pension Credit. + + ``TOTCAPB3`` is DWP's derived benefit-unit total of the adults' savings and + investments (current, savings and NS&I accounts, gilts, unit and investment + trusts, shares and bonds, ISAs, credit unions), the measure its FRS-based + National Statistics use. Pension Credit counts the claimant's capital and, + under the State Pension Credit Act 2002 s. 5, the partner's, which is what + this records. The household wealth imputation instead draws a household's + wealth from Wealth and Assets Survey households with similar income, + composition, tenure and region, with no information on means-tested + receipt, and policyengine-uk spreads it over the household's pension-age + adults. + + ``TOTCAPB3`` covers financial assets only: second homes and land, which + Pension Credit also counts, are not in it. A missing or negative value gives + -1, so policyengine-uk falls back to the household proxy. + """ + if "totcapb3" not in benunit.columns: + return np.full(len(benunit), -1.0) + capital = pd.to_numeric(benunit["totcapb3"], errors="coerce").to_numpy(dtype=float) + return np.where(np.isfinite(capital) & (capital >= 0), capital, -1.0) + + def create_frs( raw_frs_folder: str, year: int, @@ -1627,6 +1651,13 @@ def _reported_benunit_mask(person_column: str) -> np.ndarray: pe_benunit["is_married"] = benunit.famtypb2.isin([5, 7]) + # Pension Credit capital as the FRS records it for the benefit unit, in + # place of the household wealth proxy (policyengine-uk + # `pension_credit_reported_capital`). + pe_benunit["pension_credit_reported_capital"] = ( + derive_pension_credit_reported_capital(benunit) + ) + # Assign property_purchased to a share of households matching the UK # housing transaction rate, so only genuine purchasers are charged SDLT. # diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py new file mode 100644 index 00000000..ca42cf43 --- /dev/null +++ b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py @@ -0,0 +1,44 @@ +"""`pension_credit_reported_capital` from the FRS benefit-unit capital +measure (TOTCAPB3). + +Invariants: +1. A finite, non-negative TOTCAPB3 is carried over unchanged. +2. A missing, non-numeric or negative value gives -1 (policyengine-uk's + "none recorded" sentinel), so the household proxy applies. +3. Without a TOTCAPB3 column every benefit unit gets -1. +4. The output is always -1 or a non-negative number, one per benefit unit. +""" + +import numpy as np +import pandas as pd + +from policyengine_uk_data.datasets.frs import derive_pension_credit_reported_capital + + +def test_values_carry_over_and_invalid_values_fall_back(): + benunit = pd.DataFrame( + {"totcapb3": [0.0, 300.0, 2_900.0, 1_250_000.0, np.nan, -5.0, "x"]} + ) + result = derive_pension_credit_reported_capital(benunit) + assert result.tolist() == [0.0, 300.0, 2_900.0, 1_250_000.0, -1.0, -1.0, -1.0] + + +def test_missing_column_gives_sentinel(): + benunit = pd.DataFrame({"benunit_id": [101, 102, 201]}) + assert derive_pension_credit_reported_capital(benunit).tolist() == [-1.0] * 3 + + +def test_output_is_sentinel_or_non_negative_for_random_inputs(): + rng = np.random.default_rng(1_792) + for _ in range(50): + n = int(rng.integers(1, 200)) + values = rng.normal(5_000, 20_000, n) + values[rng.random(n) < 0.1] = np.nan + result = derive_pension_credit_reported_capital( + pd.DataFrame({"totcapb3": values}) + ) + assert len(result) == n + assert np.all((result == -1) | (result >= 0)) + keep = np.isfinite(values) & (values >= 0) + np.testing.assert_array_equal(result[keep], values[keep]) + assert np.all(result[~keep] == -1) From cb194e4f7ca7c8f0824fd59bce49c73b33b2545c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 2 Oct 2026 10:12:28 -0400 Subject: [PATCH 2/5] Clear FRS capital on SPI copies, uprate it with savings, require PE-UK 2.107.0 - SPI-synthetic copies get pension_credit_reported_capital = -1: the FRS donor's capital does not belong with SPI-imputed incomes (24 of 152 SPI PC reporters were entitled in build E only because of it). - uprating_factors.csv and uprating_growth_factors.csv get the column with the savings row (policyengine-uk uprates it with the same per-capita GDP index), so uprate_dataset keeps the two in step. - policyengine-uk >= 2.107.0, the first release defining the variable (#2018); core 3.32.12 comes with it. Co-Authored-By: Claude Opus 5.5 --- .../datasets/imputations/income.py | 14 ++++++++ .../storage/uprating_factors.csv | 1 + .../storage/uprating_growth_factors.csv | 1 + .../test_pension_credit_reported_capital.py | 35 +++++++++++++++++++ pyproject.toml | 2 +- uv.lock | 16 ++++----- 6 files changed, 60 insertions(+), 9 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index feafcb27..bd5c1d11 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -234,6 +234,19 @@ def impute_over_incomes( return dataset +def clear_frs_reported_capital(dataset: UKSingleYearDataset) -> UKSingleYearDataset: + """Set ``pension_credit_reported_capital`` to -1 (none recorded). + + Used on the SPI-synthetic copy. The FRS benefit-unit capital belongs to the + FRS donor, whose incomes the SPI imputation replaces; keeping it would + assess an SPI-income unit on the donor's capital. With -1, policyengine-uk + uses the household capital proxy for these rows. + """ + if "pension_credit_reported_capital" in dataset.benunit.columns: + dataset.benunit["pension_credit_reported_capital"] = -1.0 + return dataset + + def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: """ Impute detailed income components using trained model. @@ -262,6 +275,7 @@ def impute_income(dataset: UKSingleYearDataset) -> UKSingleYearDataset: zero_weight_copy = dataset.copy() zero_weight_copy.household.household_weight = 0 zero_weight_copy.household["household_is_spi_synthetic"] = True + zero_weight_copy = clear_frs_reported_capital(zero_weight_copy) zero_weight_copy = subsample_dataset(zero_weight_copy, 10_000) model = create_income_model() diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index dacd591b..8453aaa9 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -54,6 +54,7 @@ other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1. other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +pension_credit_reported_capital,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 122b4ed6..6722cb58 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -54,6 +54,7 @@ other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +pension_credit_reported_capital,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py index ca42cf43..0081769b 100644 --- a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py +++ b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py @@ -42,3 +42,38 @@ def test_output_is_sentinel_or_non_negative_for_random_inputs(): keep = np.isfinite(values) & (values >= 0) np.testing.assert_array_equal(result[keep], values[keep]) assert np.all(result[~keep] == -1) + + +def test_uprating_rows_match_savings(): + """The column grows with ``savings`` (policyengine-uk uprates it with the + same per-capita GDP index), so the build's uprating keeps them in step.""" + from policyengine_uk_data.storage import STORAGE_FOLDER + + for name in ["uprating_factors.csv", "uprating_growth_factors.csv"]: + table = pd.read_csv(STORAGE_FOLDER / name).set_index("Variable") + pd.testing.assert_series_equal( + table.loc["pension_credit_reported_capital"], + table.loc["savings"], + check_names=False, + ) + + +def test_spi_copy_records_no_capital(): + """SPI-synthetic copies carry SPI-imputed incomes, so the FRS donor's + capital is cleared to -1 (household proxy) on them.""" + from types import SimpleNamespace + + from policyengine_uk_data.datasets.imputations.income import ( + clear_frs_reported_capital, + ) + + copy = SimpleNamespace( + benunit=pd.DataFrame({"pension_credit_reported_capital": [0.0, 300.0, -1.0]}) + ) + assert clear_frs_reported_capital(copy).benunit[ + "pension_credit_reported_capital" + ].tolist() == [-1.0, -1.0, -1.0] + without = SimpleNamespace(benunit=pd.DataFrame({"benunit_id": [1, 2]})) + assert "pension_credit_reported_capital" not in ( + clear_frs_reported_capital(without).benunit.columns + ) diff --git a/pyproject.toml b/pyproject.toml index beff7f1a..8b87d71d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "policyengine", "google-cloud-storage", "google-auth", - "policyengine-uk>=2.93.0", + "policyengine-uk>=2.107.0", "microcalibrate>=0.18.0", "microimpute>=1.0.1", "ruff>=0.9.0", diff --git a/uv.lock b/uv.lock index a3e9f44d..ab6e353e 100644 --- a/uv.lock +++ b/uv.lock @@ -1322,7 +1322,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.31.1" +version = "3.32.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1342,14 +1342,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9f/1e/cd58cb6947720a5b7850c06a988abbc0eef53e2be192ed49a17ed62d78d4/policyengine_core-3.31.1.tar.gz", hash = "sha256:3ca8f666c3e67b0c0ce992fa3dc9d1ff2c095a3860f3d719883588b2f9e44ec3", size = 500450, upload-time = "2026-08-28T05:06:43.16Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/cd/2e9c7e4a0c4395a0d556cebde1fa92b94c4c2c594815938e5bfd052b3b47/policyengine_core-3.32.12.tar.gz", hash = "sha256:1d163936b3e4b2704cf0fe33eee3b93832a84aa4bf027b552a745d6f6d35bad8", size = 451537, upload-time = "2026-10-02T04:58:58.42Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/74/ce/995992049664b6b4cecd1e4db76b143e07aaee8239043057436f48ccbe75/policyengine_core-3.31.1-py3-none-any.whl", hash = "sha256:308200b28aa26bb79d7f0badebac391c0a81704d45ba4ef48ac79e88f0b21e6b", size = 245719, upload-time = "2026-08-28T05:06:41.769Z" }, + { url = "https://files.pythonhosted.org/packages/36/9e/f8f41e2f33454be51e9116c1b96db77ec9982d5d97d8691a0d8bd7b35b99/policyengine_core-3.32.12-py3-none-any.whl", hash = "sha256:44f0cdc100ff171f107671965bdbb41aa0c9965f632e400734e9eb2bf6ed80ee", size = 250582, upload-time = "2026-10-02T04:58:56.986Z" }, ] [[package]] name = "policyengine-uk" -version = "2.93.0" +version = "2.107.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1357,14 +1357,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/af/04/ab8fbb526c22d5c422eed374244d977b123241baa6f96a604c600876647a/policyengine_uk-2.93.0.tar.gz", hash = "sha256:e3b8eab824e7dad3a1fd61fa8503442a8b1b2285e05fff7ab60f93b6a9b22578", size = 1246786, upload-time = "2026-08-28T14:55:16.266Z" } +sdist = { url = "https://files.pythonhosted.org/packages/13/d6/cb79e5e93b9209e9af1b3962db951b7655af3d1a3d53f20a15c7208330e4/policyengine_uk-2.107.0.tar.gz", hash = "sha256:e170c7539a891a91405dba8f1bbc534d54c70283b6ff685661f891c8df0b821c", size = 1448364, upload-time = "2026-10-02T13:38:49.609Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/74/5e626f9f80b9485b10bcccbef8d93a635abe309bfdfad437b5de38a80fae/policyengine_uk-2.93.0-py3-none-any.whl", hash = "sha256:4f95feffa3b27fab927f9b903f8a9f1f0d63b2fba209d6b8ac010f0f07a5ec7c", size = 2046525, upload-time = "2026-08-28T14:55:14.114Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1e/195c05d73e563f08ad14dff322f35efa09ef7b168c1732b3ff30d66c8b35/policyengine_uk-2.107.0-py3-none-any.whl", hash = "sha256:1ce84c3edefd4e07ac5b9d9254448ad3f0aada0694212c543b4ba3a6106b1973", size = 2399214, upload-time = "2026-10-02T13:38:47.79Z" }, ] [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1419,7 +1419,7 @@ requires-dist = [ { name = "pandas" }, { name = "policyengine" }, { name = "policyengine-core", specifier = ">=3.19.4" }, - { name = "policyengine-uk", specifier = ">=2.93.0" }, + { name = "policyengine-uk", specifier = ">=2.107.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pyyaml" }, From dfe8dcaa175cd7df576c08bca1fc483ba016439b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 17:22:26 -0400 Subject: [PATCH 3/5] Use TOTCAPB4, model-derived uprating rows and core 3.32.13 (review of #513) - TOTCAPB4 (DWP's below-average-resources measure since 2019/20) in place of TOTCAPB3, which stays as the fallback for earlier survey years. On #510 build D plus the column: the same 87.7k non-entitled reporters as with TOTCAPB3, GB entitled units 2,080.7k vs 2,083.5k, gate claims 1,290.0k vs 1,291.8k. The docstring now calls it an approximation (financial assets only; no Sch V disregard or reg 19 valuation). - uprating_factors.csv and uprating_growth_factors.csv rows recomputed from policyengine-uk 2.107.0's per-capita GDP index (2024 1.204, 2025 1.256), not copied from the stale savings row, so calibration and runtime grow the capital alike; the test now checks the rows against the model. - policyengine-core 3.32.13 in uv.lock (3.32.12 leaks memory per simulation). Co-Authored-By: Claude Opus 5.5 --- .../frs-pc-reported-capital.changed.md | 2 +- policyengine_uk_data/datasets/frs.py | 42 +++++++------ .../storage/uprating_factors.csv | 2 +- .../storage/uprating_growth_factors.csv | 2 +- .../test_pension_credit_reported_capital.py | 59 ++++++++++++++----- uv.lock | 6 +- 6 files changed, 73 insertions(+), 40 deletions(-) diff --git a/changelog.d/frs-pc-reported-capital.changed.md b/changelog.d/frs-pc-reported-capital.changed.md index 7986dc73..d6433759 100644 --- a/changelog.d/frs-pc-reported-capital.changed.md +++ b/changelog.d/frs-pc-reported-capital.changed.md @@ -1 +1 @@ -Carry each benefit unit's FRS total capital (TOTCAPB3) into policyengine-uk's `pension_credit_reported_capital`, so Pension Credit's capital test uses the survey's own benefit-unit capital instead of imputed household wealth. +Carry each benefit unit's FRS total capital (TOTCAPB4, DWP's current benefit-unit savings and investments measure; TOTCAPB3 for earlier survey years) into policyengine-uk's `pension_credit_reported_capital`, so Pension Credit's capital test uses the survey's own benefit-unit capital instead of imputed household wealth. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 5cdb760c..4f69a126 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -573,25 +573,31 @@ def validate_frs_survey_year(raw_frs_folder, year: int) -> None: def derive_pension_credit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: """Each benefit unit's capital as the FRS records it, for Pension Credit. - ``TOTCAPB3`` is DWP's derived benefit-unit total of the adults' savings and - investments (current, savings and NS&I accounts, gilts, unit and investment - trusts, shares and bonds, ISAs, credit unions), the measure its FRS-based - National Statistics use. Pension Credit counts the claimant's capital and, - under the State Pension Credit Act 2002 s. 5, the partner's, which is what - this records. The household wealth imputation instead draws a household's - wealth from Wealth and Assets Survey households with similar income, - composition, tenure and region, with no information on means-tested - receipt, and policyengine-uk spreads it over the household's pension-age - adults. - - ``TOTCAPB3`` covers financial assets only: second homes and land, which - Pension Credit also counts, are not in it. A missing or negative value gives - -1, so policyengine-uk falls back to the household proxy. + Uses ``TOTCAPB4``, DWP's derived benefit-unit total of the adults' savings + and investments, which its below-average-resources statistics use in place + of ``TOTCAPB3`` since it became available in 2019/20; ``TOTCAPB3`` is the + fallback for earlier survey years. Pension Credit counts the claimant's + capital and, under the State Pension Credit Act 2002 s. 5, the partner's, + and this is a benefit-unit measure. It is an approximation of Pension + Credit capital, not the assessed figure: it covers financial assets only + (second homes and land, which Pension Credit also counts, are not in it), + and no Schedule V disregard or reg. 19 valuation is applied to it. The + household wealth imputation instead draws a household's wealth from Wealth + and Assets Survey households with similar income, composition, tenure and + region, with no information on means-tested receipt, and policyengine-uk + spreads it over the household's pension-age adults. + + A missing or negative value gives -1, so policyengine-uk falls back to the + household proxy. """ - if "totcapb3" not in benunit.columns: - return np.full(len(benunit), -1.0) - capital = pd.to_numeric(benunit["totcapb3"], errors="coerce").to_numpy(dtype=float) - return np.where(np.isfinite(capital) & (capital >= 0), capital, -1.0) + capital = pd.Series(np.nan, index=benunit.index, dtype=float) + for column in ("totcapb3", "totcapb4"): # later columns take precedence + if column in benunit.columns: + values = pd.to_numeric(benunit[column], errors="coerce") + valid = np.isfinite(values) & (values >= 0) + capital = capital.where(~valid, values) + values = capital.to_numpy(dtype=float) + return np.where(np.isfinite(values) & (values >= 0), values, -1.0) def create_frs( diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 8453aaa9..149ef19f 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -54,7 +54,7 @@ other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1. other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -pension_credit_reported_capital,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +pension_credit_reported_capital,1.0,1.0,1.102,1.161,1.204,1.256,1.293,1.335,1.376,1.417,1.462,1.508,1.555,1.604,1.655 pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 6722cb58..956f740d 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -54,7 +54,7 @@ other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -pension_credit_reported_capital,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +pension_credit_reported_capital,0,0.0,0.102,0.054,0.037,0.043,0.029,0.032,0.031,0.03,0.032,0.031,0.031,0.032,0.032 pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py index 0081769b..2e93839d 100644 --- a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py +++ b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py @@ -1,11 +1,13 @@ """`pension_credit_reported_capital` from the FRS benefit-unit capital -measure (TOTCAPB3). +measure (TOTCAPB4, falling back to TOTCAPB3). Invariants: -1. A finite, non-negative TOTCAPB3 is carried over unchanged. -2. A missing, non-numeric or negative value gives -1 (policyengine-uk's - "none recorded" sentinel), so the household proxy applies. -3. Without a TOTCAPB3 column every benefit unit gets -1. +1. A finite, non-negative TOTCAPB4 is carried over unchanged; where TOTCAPB4 + is absent or invalid, a valid TOTCAPB3 is used. +2. A missing, non-numeric or negative value in both gives -1 + (policyengine-uk's "none recorded" sentinel), so the household proxy + applies. +3. Without either column every benefit unit gets -1. 4. The output is always -1 or a non-negative number, one per benefit unit. """ @@ -23,6 +25,17 @@ def test_values_carry_over_and_invalid_values_fall_back(): assert result.tolist() == [0.0, 300.0, 2_900.0, 1_250_000.0, -1.0, -1.0, -1.0] +def test_totcapb4_takes_precedence_with_totcapb3_fallback(): + benunit = pd.DataFrame( + { + "totcapb3": [100.0, 200.0, 300.0, np.nan, -1.0], + "totcapb4": [150.0, np.nan, -5.0, 400.0, np.nan], + } + ) + result = derive_pension_credit_reported_capital(benunit) + assert result.tolist() == [150.0, 200.0, 300.0, 400.0, -1.0] + + def test_missing_column_gives_sentinel(): benunit = pd.DataFrame({"benunit_id": [101, 102, 201]}) assert derive_pension_credit_reported_capital(benunit).tolist() == [-1.0] * 3 @@ -35,7 +48,7 @@ def test_output_is_sentinel_or_non_negative_for_random_inputs(): values = rng.normal(5_000, 20_000, n) values[rng.random(n) < 0.1] = np.nan result = derive_pension_credit_reported_capital( - pd.DataFrame({"totcapb3": values}) + pd.DataFrame({"totcapb4": values}) ) assert len(result) == n assert np.all((result == -1) | (result >= 0)) @@ -44,18 +57,32 @@ def test_output_is_sentinel_or_non_negative_for_random_inputs(): assert np.all(result[~keep] == -1) -def test_uprating_rows_match_savings(): - """The column grows with ``savings`` (policyengine-uk uprates it with the - same per-capita GDP index), so the build's uprating keeps them in step.""" - from policyengine_uk_data.storage import STORAGE_FOLDER +def test_uprating_rows_match_the_model(): + """The build uprates the column with these rows (calibration materialises + the calibration year from them), and policyengine-uk projects the saved + dataset with the variable's own uprating index at runtime. The rows must + equal what ``create_policyengine_uprating_factors_table`` derives from the + locked policyengine-uk, or a unit near the 10,000 pound deemed-income + disregard can be assessed differently in calibration and at runtime.""" + from policyengine_uk.system import system - for name in ["uprating_factors.csv", "uprating_growth_factors.csv"]: - table = pd.read_csv(STORAGE_FOLDER / name).set_index("Variable") - pd.testing.assert_series_equal( - table.loc["pension_credit_reported_capital"], - table.loc["savings"], - check_names=False, + from policyengine_uk_data.storage import STORAGE_FOLDER + from policyengine_uk_data.utils.uprating import END_YEAR, START_YEAR + + variable = system.variables["pension_credit_reported_capital"] + index = system.parameters.get_child(variable.uprating) + years = range(START_YEAR, END_YEAR + 1) + expected = {y: round(index(y) / index(START_YEAR), 3) for y in years} + factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable") + growth = pd.read_csv(STORAGE_FOLDER / "uprating_growth_factors.csv").set_index( + "Variable" + ) + for y in years: + assert factors.loc["pension_credit_reported_capital", str(y)] == expected[y] + expected_growth = ( + 0 if y == START_YEAR else round(expected[y] / expected[y - 1] - 1, 3) ) + assert growth.loc["pension_credit_reported_capital", str(y)] == expected_growth def test_spi_copy_records_no_capital(): diff --git a/uv.lock b/uv.lock index ab6e353e..b0b2249b 100644 --- a/uv.lock +++ b/uv.lock @@ -1322,7 +1322,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.32.12" +version = "3.32.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1342,9 +1342,9 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e3/cd/2e9c7e4a0c4395a0d556cebde1fa92b94c4c2c594815938e5bfd052b3b47/policyengine_core-3.32.12.tar.gz", hash = "sha256:1d163936b3e4b2704cf0fe33eee3b93832a84aa4bf027b552a745d6f6d35bad8", size = 451537, upload-time = "2026-10-02T04:58:58.42Z" } +sdist = { url = "https://files.pythonhosted.org/packages/40/0a/b6c27953e1083d3ab2c40330c49580130fff48a2fe9cbfc8d731e92882e6/policyengine_core-3.32.13.tar.gz", hash = "sha256:a870f7e212fdfa1b5fdad7df92aa2b325fd0280ede16a40aeeceea74d44fb864", size = 457899, upload-time = "2026-10-03T11:30:20.572Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/36/9e/f8f41e2f33454be51e9116c1b96db77ec9982d5d97d8691a0d8bd7b35b99/policyengine_core-3.32.12-py3-none-any.whl", hash = "sha256:44f0cdc100ff171f107671965bdbb41aa0c9965f632e400734e9eb2bf6ed80ee", size = 250582, upload-time = "2026-10-02T04:58:56.986Z" }, + { url = "https://files.pythonhosted.org/packages/d8/01/00ebf4c13152347bc5396b40e80445fa92ea96712b9ec0bfdca7c5d11413/policyengine_core-3.32.13-py3-none-any.whl", hash = "sha256:262543c3ac696207f304f55664711b72e894fbad7b566407afa9d30b16086a41", size = 251251, upload-time = "2026-10-03T11:30:19.171Z" }, ] [[package]] From 1cecceeaea9e701bb02b68207653dddae9f0bdfd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 3 Oct 2026 18:45:24 -0400 Subject: [PATCH 4/5] Check PolicyBench couples against claimant and partner (policyengine-uk 2.107) policyengine-uk 2.107 (#1896) presumes a couple married when the dataset does not say, and its couple is the claimant and partner rather than any two adults: a member under 20 and 16+ years younger than the claimant is presumed to be their child. The family-structure test compared is_married with the adult count, which the lock bump to 2.107.0 made fail; it now compares with the claimant-and-partner count. family_type still counts is_adult, so that check is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_policybench_transfer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/policyengine_uk_data/tests/test_policybench_transfer.py b/policyengine_uk_data/tests/test_policybench_transfer.py index 900b73d1..9a8d587c 100644 --- a/policyengine_uk_data/tests/test_policybench_transfer.py +++ b/policyengine_uk_data/tests/test_policybench_transfer.py @@ -205,6 +205,9 @@ def test_policybench_transfer_family_structure_matches_person_membership( person_benunit_ids = sim.calculate("person_benunit_id", map_to="person").values is_adult = sim.calculate("is_adult", map_to="person").values is_child = sim.calculate("is_child", map_to="person").values + is_claimant_or_partner = sim.calculate( + "is_claimant_or_partner", map_to="person" + ).values is_married = sim.calculate("is_married", map_to="benunit").values family_type = sim.calculate("family_type", map_to="benunit").values @@ -213,7 +216,12 @@ def test_policybench_transfer_family_structure_matches_person_membership( adults = int(is_adult[member_mask].sum()) children = int(is_child[member_mask].sum()) - assert bool(married) == (adults == 2) + # policyengine-uk (from 2.107, #1896) presumes a couple married when + # the dataset does not say, and a couple is a claimant and partner, + # not any two adults: a member under 20 and 16+ years younger than + # the claimant is presumed to be their child. + claimant_and_partner = int(is_claimant_or_partner[member_mask].sum()) + assert bool(married) == (claimant_and_partner == 2) if adults == 2 and children > 0: expected = "COUPLE_WITH_CHILDREN" From a1690afaa72595b2d5a847ac6581b692be1fbf14 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 06:32:29 -0400 Subject: [PATCH 5/5] Keep the TOTCAPB3 fallback with nullable inputs (review of #513) With pandas nullable Float64 columns, a missing TOTCAPB4 (pd.NA) made the validity mask NA and wiped a valid TOTCAPB3. Convert each column to plain float64 first so the mask has no missing entries. The FRS reader uses plain NumPy dtypes, so this release's input is unaffected. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/frs.py | 15 ++++++++++----- .../tests/test_pension_credit_reported_capital.py | 13 +++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 4f69a126..77e6f339 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -590,14 +590,19 @@ def derive_pension_credit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: A missing or negative value gives -1, so policyengine-uk falls back to the household proxy. """ - capital = pd.Series(np.nan, index=benunit.index, dtype=float) + capital = np.full(len(benunit), np.nan) for column in ("totcapb3", "totcapb4"): # later columns take precedence if column in benunit.columns: - values = pd.to_numeric(benunit[column], errors="coerce") + # Plain float64, so nullable (pd.NA) inputs become NaN and the + # validity mask is a plain bool array with no missing entries. + values = ( + pd.to_numeric(benunit[column], errors="coerce") + .astype("float64") + .to_numpy(dtype=float, na_value=np.nan) + ) valid = np.isfinite(values) & (values >= 0) - capital = capital.where(~valid, values) - values = capital.to_numpy(dtype=float) - return np.where(np.isfinite(values) & (values >= 0), values, -1.0) + capital = np.where(valid, values, capital) + return np.where(np.isfinite(capital) & (capital >= 0), capital, -1.0) def create_frs( diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py index 2e93839d..d1918581 100644 --- a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py +++ b/policyengine_uk_data/tests/test_pension_credit_reported_capital.py @@ -36,6 +36,19 @@ def test_totcapb4_takes_precedence_with_totcapb3_fallback(): assert result.tolist() == [150.0, 200.0, 300.0, 400.0, -1.0] +def test_nullable_inputs_keep_the_totcapb3_fallback(): + """Nullable (pd.NA) columns must not wipe a valid TOTCAPB3 where TOTCAPB4 + is missing.""" + benunit = pd.DataFrame( + { + "totcapb3": pd.array([0.0, 250.0, pd.NA, -3.0], dtype="Float64"), + "totcapb4": pd.array([pd.NA, pd.NA, 400.0, pd.NA], dtype="Float64"), + } + ) + result = derive_pension_credit_reported_capital(benunit) + assert result.tolist() == [0.0, 250.0, 400.0, -1.0] + + def test_missing_column_gives_sentinel(): benunit = pd.DataFrame({"benunit_id": [101, 102, 201]}) assert derive_pension_credit_reported_capital(benunit).tolist() == [-1.0] * 3