diff --git a/changelog.d/frs-benunit-reported-capital.changed.md b/changelog.d/frs-benunit-reported-capital.changed.md new file mode 100644 index 00000000..db1f50e7 --- /dev/null +++ b/changelog.d/frs-benunit-reported-capital.changed.md @@ -0,0 +1 @@ +- Fill policyengine-uk's `benunit_reported_capital` with each benefit unit's FRS total capital (TOTCAPB4; TOTCAPB3 for earlier survey years) in place of `pension_credit_reported_capital`. policyengine-uk's Universal Credit and Pension Credit reported capital both default to it, so Universal Credit's capital test now uses the FRS's own benefit-unit capital instead of the household wealth imputation, as Pension Credit already did. SPI-synthetic copies still get -1 (household proxy). diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 711127b5..32074b31 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -790,22 +790,25 @@ def validate_frs_survey_year(raw_frs_folder, year: int) -> None: ) -def derive_pension_credit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: - """Each benefit unit's capital as the FRS records it, for Pension Credit. +def derive_benunit_reported_capital(benunit: pd.DataFrame) -> np.ndarray: + """Each benefit unit's capital as the FRS records it. Uses ``TOTCAPB4``, DWP's derived benefit-unit total of the adults' savings and investments, which its below-average-resources statistics use in place of ``TOTCAPB3`` since it became available in 2019/20; ``TOTCAPB3`` is the - fallback for earlier survey years. Pension Credit counts the claimant's - capital and, under the State Pension Credit Act 2002 s. 5, the partner's, - and this is a benefit-unit measure. It is an approximation of Pension - Credit capital, not the assessed figure: it covers financial assets only - (second homes and land, which Pension Credit also counts, are not in it), - and no Schedule V disregard or reg. 19 valuation is applied to it. The - household wealth imputation instead draws a household's wealth from Wealth - and Assets Survey households with similar income, composition, tenure and - region, with no information on means-tested receipt, and policyengine-uk - spreads it over the household's pension-age adults. + fallback for earlier survey years. This fills policyengine-uk's + ``benunit_reported_capital``, an input that records the observed figure + only. Each means test derives its own capital from it: Universal Credit + (Welfare Reform Act 2012 s. 5) and Pension Credit (State Pension Credit + Act 2002 s. 5) both assess the claimant's and partner's capital together, + which this benefit-unit measure covers. As a means-test figure it is an + approximation, not the assessed amount: it covers financial assets only + (second homes and land, which both tests also count, are not in it), and + no disregard or valuation rule is applied to it. The household wealth + imputation instead draws a household's wealth from Wealth and Assets + Survey households with similar income, composition, tenure and region, + with no information on means-tested receipt, and policyengine-uk shares + it out between the household's benefit units. A missing or negative value gives -1, so policyengine-uk falls back to the household proxy. @@ -1843,12 +1846,11 @@ def determine_education_level(fted_val, typeed2_val, age_val): pe_benunit["is_married"] = benunit.famtypb2.isin([5, 7]) - # Pension Credit capital as the FRS records it for the benefit unit, in - # place of the household wealth proxy (policyengine-uk - # `pension_credit_reported_capital`). - pe_benunit["pension_credit_reported_capital"] = ( - derive_pension_credit_reported_capital(benunit) - ) + # The benefit unit's capital as the FRS records it. policyengine-uk's + # Universal Credit and Pension Credit capital (`uc_reported_capital`, + # `pension_credit_reported_capital`) default to it in place of the + # household wealth proxy. + pe_benunit["benunit_reported_capital"] = derive_benunit_reported_capital(benunit) # Assign property_purchased to a share of households matching the UK # housing transaction rate, so only genuine purchasers are charged SDLT. diff --git a/policyengine_uk_data/datasets/imputations/income.py b/policyengine_uk_data/datasets/imputations/income.py index 799b5f35..76cfd2b8 100644 --- a/policyengine_uk_data/datasets/imputations/income.py +++ b/policyengine_uk_data/datasets/imputations/income.py @@ -461,16 +461,24 @@ def impute_over_incomes( return dataset +REPORTED_CAPITAL_COLUMNS = ( + "benunit_reported_capital", + "pension_credit_reported_capital", + "uc_reported_capital", +) + + def clear_frs_reported_capital(dataset: UKSingleYearDataset) -> UKSingleYearDataset: - """Set ``pension_credit_reported_capital`` to -1 (none recorded). + """Set the FRS-recorded capital columns to -1 (none recorded). Used on the SPI-synthetic copy. The FRS benefit-unit capital belongs to the FRS donor, whose incomes the SPI imputation replaces; keeping it would assess an SPI-income unit on the donor's capital. With -1, policyengine-uk - uses the household capital proxy for these rows. + uses the household capital proxy for these rows in every means test. """ - if "pension_credit_reported_capital" in dataset.benunit.columns: - dataset.benunit["pension_credit_reported_capital"] = -1.0 + for column in REPORTED_CAPITAL_COLUMNS: + if column in dataset.benunit.columns: + dataset.benunit[column] = -1.0 return dataset diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 149ef19f..946afa40 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -2,6 +2,7 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2 afcs_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 alcohol_and_tobacco_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 benunit_rent,1.0,1.0,1.0,1.11,1.184,1.223,1.275,1.312,1.351,1.392,1.392,1.392,1.392,1.392,1.392 +benunit_reported_capital,1.0,1.0,1.102,1.161,1.204,1.256,1.293,1.335,1.376,1.417,1.462,1.508,1.555,1.604,1.655 bsp_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 capital_gains,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 capital_gains_before_response,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 @@ -54,7 +55,6 @@ other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1. other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 owned_land,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 -pension_credit_reported_capital,1.0,1.0,1.102,1.161,1.204,1.256,1.293,1.335,1.376,1.417,1.462,1.508,1.555,1.604,1.655 pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 956f740d..5bf1647c 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -2,6 +2,7 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2 afcs_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 alcohol_and_tobacco_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 benunit_rent,0,0.0,0.0,0.11,0.067,0.033,0.043,0.029,0.03,0.03,0.0,0.0,0.0,0.0,0.0 +benunit_reported_capital,0,0.0,0.102,0.054,0.037,0.043,0.029,0.032,0.031,0.03,0.032,0.031,0.031,0.032,0.032 bsp_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 capital_gains,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 capital_gains_before_response,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 @@ -54,7 +55,6 @@ other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 owned_land,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 -pension_credit_reported_capital,0,0.0,0.102,0.054,0.037,0.043,0.029,0.032,0.031,0.03,0.032,0.031,0.031,0.032,0.032 pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py b/policyengine_uk_data/tests/test_benunit_reported_capital.py similarity index 61% rename from policyengine_uk_data/tests/test_pension_credit_reported_capital.py rename to policyengine_uk_data/tests/test_benunit_reported_capital.py index d1918581..16c9d06f 100644 --- a/policyengine_uk_data/tests/test_pension_credit_reported_capital.py +++ b/policyengine_uk_data/tests/test_benunit_reported_capital.py @@ -1,5 +1,6 @@ -"""`pension_credit_reported_capital` from the FRS benefit-unit capital -measure (TOTCAPB4, falling back to TOTCAPB3). +"""`benunit_reported_capital` from the FRS benefit-unit capital measure +(TOTCAPB4, falling back to TOTCAPB3). policyengine-uk's Universal Credit and +Pension Credit reported capital default to it. Invariants: 1. A finite, non-negative TOTCAPB4 is carried over unchanged; where TOTCAPB4 @@ -14,14 +15,14 @@ import numpy as np import pandas as pd -from policyengine_uk_data.datasets.frs import derive_pension_credit_reported_capital +from policyengine_uk_data.datasets.frs import derive_benunit_reported_capital def test_values_carry_over_and_invalid_values_fall_back(): benunit = pd.DataFrame( {"totcapb3": [0.0, 300.0, 2_900.0, 1_250_000.0, np.nan, -5.0, "x"]} ) - result = derive_pension_credit_reported_capital(benunit) + result = derive_benunit_reported_capital(benunit) assert result.tolist() == [0.0, 300.0, 2_900.0, 1_250_000.0, -1.0, -1.0, -1.0] @@ -32,7 +33,7 @@ def test_totcapb4_takes_precedence_with_totcapb3_fallback(): "totcapb4": [150.0, np.nan, -5.0, 400.0, np.nan], } ) - result = derive_pension_credit_reported_capital(benunit) + result = derive_benunit_reported_capital(benunit) assert result.tolist() == [150.0, 200.0, 300.0, 400.0, -1.0] @@ -45,13 +46,13 @@ def test_nullable_inputs_keep_the_totcapb3_fallback(): "totcapb4": pd.array([pd.NA, pd.NA, 400.0, pd.NA], dtype="Float64"), } ) - result = derive_pension_credit_reported_capital(benunit) + result = derive_benunit_reported_capital(benunit) assert result.tolist() == [0.0, 250.0, 400.0, -1.0] def test_missing_column_gives_sentinel(): benunit = pd.DataFrame({"benunit_id": [101, 102, 201]}) - assert derive_pension_credit_reported_capital(benunit).tolist() == [-1.0] * 3 + assert derive_benunit_reported_capital(benunit).tolist() == [-1.0] * 3 def test_output_is_sentinel_or_non_negative_for_random_inputs(): @@ -60,9 +61,7 @@ def test_output_is_sentinel_or_non_negative_for_random_inputs(): n = int(rng.integers(1, 200)) values = rng.normal(5_000, 20_000, n) values[rng.random(n) < 0.1] = np.nan - result = derive_pension_credit_reported_capital( - pd.DataFrame({"totcapb4": values}) - ) + result = derive_benunit_reported_capital(pd.DataFrame({"totcapb4": values})) assert len(result) == n assert np.all((result == -1) | (result >= 0)) keep = np.isfinite(values) & (values >= 0) @@ -75,14 +74,16 @@ def test_uprating_rows_match_the_model(): the calibration year from them), and policyengine-uk projects the saved dataset with the variable's own uprating index at runtime. The rows must equal what ``create_policyengine_uprating_factors_table`` derives from the - locked policyengine-uk, or a unit near the 10,000 pound deemed-income - disregard can be assessed differently in calibration and at runtime.""" + locked policyengine-uk, or a unit near a capital threshold (Pension + Credit's 10,000 pound deemed-income disregard, Universal Credit's 6,000 + pound tariff threshold and 16,000 pound limit) can be assessed differently + in calibration and at runtime.""" from policyengine_uk.system import system from policyengine_uk_data.storage import STORAGE_FOLDER from policyengine_uk_data.utils.uprating import END_YEAR, START_YEAR - variable = system.variables["pension_credit_reported_capital"] + variable = system.variables["benunit_reported_capital"] index = system.parameters.get_child(variable.uprating) years = range(START_YEAR, END_YEAR + 1) expected = {y: round(index(y) / index(START_YEAR), 3) for y in years} @@ -91,11 +92,26 @@ def test_uprating_rows_match_the_model(): "Variable" ) for y in years: - assert factors.loc["pension_credit_reported_capital", str(y)] == expected[y] + assert factors.loc["benunit_reported_capital", str(y)] == expected[y] expected_growth = ( 0 if y == START_YEAR else round(expected[y] / expected[y - 1] - 1, 3) ) - assert growth.loc["pension_credit_reported_capital", str(y)] == expected_growth + assert growth.loc["benunit_reported_capital", str(y)] == expected_growth + + +def test_programme_capital_defaults_to_the_recorded_input(): + """policyengine-uk derives both programmes' reported capital from the one + input this build fills, so the dataset stores only the observed figure and + the uprating tables carry no rows for the programme variables.""" + from policyengine_uk.system import system + + from policyengine_uk_data.storage import STORAGE_FOLDER + + assert not system.variables["benunit_reported_capital"].formulas + factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv") + for name in ("pension_credit_reported_capital", "uc_reported_capital"): + assert system.variables[name].formulas + assert name not in set(factors["Variable"]) def test_spi_copy_records_no_capital(): @@ -108,12 +124,15 @@ def test_spi_copy_records_no_capital(): ) copy = SimpleNamespace( - benunit=pd.DataFrame({"pension_credit_reported_capital": [0.0, 300.0, -1.0]}) + benunit=pd.DataFrame( + { + "benunit_reported_capital": [0.0, 300.0, -1.0], + "uc_reported_capital": [5.0, -1.0, 7.0], + } + ) ) - assert clear_frs_reported_capital(copy).benunit[ - "pension_credit_reported_capital" - ].tolist() == [-1.0, -1.0, -1.0] + cleared = clear_frs_reported_capital(copy).benunit + assert cleared["benunit_reported_capital"].tolist() == [-1.0, -1.0, -1.0] + assert cleared["uc_reported_capital"].tolist() == [-1.0, -1.0, -1.0] without = SimpleNamespace(benunit=pd.DataFrame({"benunit_id": [1, 2]})) - assert "pension_credit_reported_capital" not in ( - clear_frs_reported_capital(without).benunit.columns - ) + assert list(clear_frs_reported_capital(without).benunit.columns) == ["benunit_id"] diff --git a/pyproject.toml b/pyproject.toml index fffc84a7..72e22407 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "policyengine", "google-cloud-storage", "google-auth", - "policyengine-uk>=2.122.0", + "policyengine-uk>=2.124.0", "microcalibrate>=0.18.0", "microimpute>=1.0.1", "ruff>=0.9.0", diff --git a/uv.lock b/uv.lock index 5ae82135..2491d189 100644 --- a/uv.lock +++ b/uv.lock @@ -1391,7 +1391,7 @@ wheels = [ [[package]] name = "policyengine-core" -version = "3.32.13" +version = "3.33.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dpath" }, @@ -1411,14 +1411,14 @@ dependencies = [ { name = "standard-imghdr" }, { name = "wheel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/0a/b6c27953e1083d3ab2c40330c49580130fff48a2fe9cbfc8d731e92882e6/policyengine_core-3.32.13.tar.gz", hash = "sha256:a870f7e212fdfa1b5fdad7df92aa2b325fd0280ede16a40aeeceea74d44fb864", size = 457899, upload-time = "2026-10-03T11:30:20.572Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/dc/b6e44d9cdb9afee354f89584eebe310d2e6480f852ca51d55570eee5a648/policyengine_core-3.33.2.tar.gz", hash = "sha256:4f0420021a5439b8d3b7c86333e291b003f4be689aebfff49cb10ad1b6952ccb", size = 693096, upload-time = "2026-10-10T19:09:47.182Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/01/00ebf4c13152347bc5396b40e80445fa92ea96712b9ec0bfdca7c5d11413/policyengine_core-3.32.13-py3-none-any.whl", hash = "sha256:262543c3ac696207f304f55664711b72e894fbad7b566407afa9d30b16086a41", size = 251251, upload-time = "2026-10-03T11:30:19.171Z" }, + { url = "https://files.pythonhosted.org/packages/43/1b/9950def57dc9ec82aa2647aed02237199e347b5b7ecb898d5dfaf70014bc/policyengine_core-3.33.2-py3-none-any.whl", hash = "sha256:631e20811b74f6a7df906891efa5c78fbfd461ecefd65902b6e1456b1a0e4098", size = 301904, upload-time = "2026-10-10T19:09:45.432Z" }, ] [[package]] name = "policyengine-uk" -version = "2.122.2" +version = "2.126.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "microdf-python" }, @@ -1426,14 +1426,14 @@ dependencies = [ { name = "pydantic" }, { name = "tables" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/62/4378fd8e493df42fef30c58924dbb0f7ca8111c927302f6e60be862d67d0/policyengine_uk-2.122.2.tar.gz", hash = "sha256:0083793f1d398feef28a64db5508d5a64bcd7834feeade6cde1055d419d505ed", size = 1976779, upload-time = "2026-10-06T02:10:56.786Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/db/ddd5cb838cbac73e85fe21df09a99fc391a508d86bc1c158420990752dc8/policyengine_uk-2.126.8.tar.gz", hash = "sha256:5063920bacd5d74a32444336de638586ba1399209083d7976f813264e6a8904c", size = 2208548, upload-time = "2026-10-11T13:32:58.405Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/11/09948c900ca6d58c1050c89b9c93d8f5884244aa6ac95661ef1a2ee9eb9a/policyengine_uk-2.122.2-py3-none-any.whl", hash = "sha256:14cefcedce43ce4a37d659a0ff17ad05695de0dc8e0c5f6e2259a69944cb5558", size = 3118697, upload-time = "2026-10-06T02:10:54.896Z" }, + { url = "https://files.pythonhosted.org/packages/75/da/6416e8f3969155c5ef0623e6e6b23d32af328382248951384d995f35cb09/policyengine_uk-2.126.8-py3-none-any.whl", hash = "sha256:8ac0b48746b87bc32eea4ab06dde3646ddd3f184e735be41c9652bfe85115bce", size = 3432792, upload-time = "2026-10-11T13:32:56.413Z" }, ] [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1490,7 +1490,7 @@ requires-dist = [ { name = "pandas" }, { name = "policyengine" }, { name = "policyengine-core", specifier = ">=3.32.13" }, - { name = "policyengine-uk", specifier = ">=2.122.0" }, + { name = "policyengine-uk", specifier = ">=2.124.0" }, { name = "pydantic", specifier = ">=2.0" }, { name = "pytest", marker = "extra == 'dev'" }, { name = "pyyaml" },