diff --git a/changelog.d/frs-boarder-lodger-rent.added.md b/changelog.d/frs-boarder-lodger-rent.added.md new file mode 100644 index 00000000..813e57a7 --- /dev/null +++ b/changelog.d/frs-boarder-lodger-rent.added.md @@ -0,0 +1 @@ +Carry the rent that boarders and lodgers pay the householder (FRS CVPAY, split by CONVBL) as the person inputs `rent_paid_as_boarder` and `rent_paid_as_lodger`. diff --git a/policyengine_uk_data/datasets/frs.py b/policyengine_uk_data/datasets/frs.py index 6ae20a90..4b5fbab6 100644 --- a/policyengine_uk_data/datasets/frs.py +++ b/policyengine_uk_data/datasets/frs.py @@ -248,6 +248,41 @@ def frs_property_income(person: pd.DataFrame, household: pd.DataFrame) -> np.nda ).values +def frs_boarder_and_lodger_rent(person: pd.DataFrame) -> tuple[np.ndarray, np.ndarray]: + """Annual rent each person pays the householder as a boarder and as a lodger. + + CVPAY is the weekly rent a boarder or lodger pays the householder, held + on the payer's own adult record. The FRS asks it about each person not + related to the household reference person in the second and later + benefit units of a conventional household (question CvPay), and CONVBL + says which the person is: + + - 1, a boarder, "someone who pays you a rent for board and lodging"; + - 2, a lodger, "someone who pays you a rent for lodging, but not food". + + A positive CVPAY with any other CONVBL is classed as a lodger's. + + CVPAY is the amount after deducting any state benefits to help with rent. + The FRS derived variables BOARDER and LODGER add housing benefit paid for + the benefit unit (HBOTHAMT); they equal the unit's summed CVPAY in every + paying unit of the raw 2023-24 and 2024-25 data. + + The released data hold this rent only on the payer's record; there is + no separate variable for what the householder receives, and the FRS + gross-income derivation leaves it out. policyengine-uk works out the + householder's receipt from these two inputs. + + Weekly amounts are annualised with ``WEEKS_IN_YEAR`` (365.25 / 7), as for + every other weekly FRS amount. policyengine-uk converts annual amounts + back to weekly with 52 weeks, so a weekly amount reaches its weekly + disregards about 0.34% higher; that convention gap is not specific to + these columns. + """ + rent_paid = np.maximum(0, person.cvpay.fillna(0).values) * WEEKS_IN_YEAR + is_boarder = person.convbl.values == 1 + return rent_paid * is_boarder, rent_paid * ~is_boarder + + def derive_is_in_non_advanced_education( current_education, is_apprentice=None, @@ -1112,6 +1147,10 @@ def determine_education_level(fted_val, typeed2_val, age_val): * 52, ) pe_person["property_income"] = frs_property_income(person, household) + ( + pe_person["rent_paid_as_boarder"], + pe_person["rent_paid_as_lodger"], + ) = frs_boarder_and_lodger_rent(person) maintenance_to_self = np.maximum( pd.Series(np.where(person.mntus1 == 2, person.mntusam1, person.mntamt1)).fillna( 0 diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index dacd591b..8f2b274f 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -61,6 +61,8 @@ private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346 private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 recreation_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +rent_paid_as_boarder,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +rent_paid_as_lodger,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 restaurants_and_hotels_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 savings,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 savings_interest_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 122b4ed6..a01ad7c2 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -61,6 +61,8 @@ private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0, private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 recreation_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +rent_paid_as_boarder,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +rent_paid_as_lodger,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 restaurants_and_hotels_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 savings,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 savings_interest_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_frs_boarder_lodger_rent.py b/policyengine_uk_data/tests/test_frs_boarder_lodger_rent.py new file mode 100644 index 00000000..96e567a3 --- /dev/null +++ b/policyengine_uk_data/tests/test_frs_boarder_lodger_rent.py @@ -0,0 +1,183 @@ +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs import ( + WEEKS_IN_YEAR, + frs_boarder_and_lodger_rent, + frs_property_income, +) + +HRP, NOT_HRP = 1, 2 +BOARDER, LODGER, NEITHER, NOT_ASKED = 1, 2, 3, 0 +OWNED_OUTRIGHT = 6 + + +def make_person(people): + return pd.DataFrame(people, columns=["person_id", "convbl", "cvpay"]) + + +def test_a_boarder_pays_rent_as_a_boarder(): + boarder, lodger = frs_boarder_and_lodger_rent( + make_person([(1_001, NOT_ASKED, 0), (1_002, BOARDER, 120)]) + ) + np.testing.assert_allclose(boarder, [0, 120 * WEEKS_IN_YEAR]) + assert lodger.tolist() == [0, 0] + + +def test_a_lodger_pays_rent_as_a_lodger(): + boarder, lodger = frs_boarder_and_lodger_rent( + make_person([(1_001, NOT_ASKED, 0), (1_002, LODGER, 100)]) + ) + assert boarder.tolist() == [0, 0] + np.testing.assert_allclose(lodger, [0, 100 * WEEKS_IN_YEAR]) + + +@pytest.mark.parametrize("convbl", [NEITHER, NOT_ASKED, -1, np.nan]) +def test_rent_without_a_boarder_code_is_a_lodgers(convbl): + boarder, lodger = frs_boarder_and_lodger_rent(make_person([(1_001, convbl, 90)])) + assert boarder.tolist() == [0] + np.testing.assert_allclose(lodger, [90 * WEEKS_IN_YEAR]) + + +@pytest.mark.parametrize("cvpay", [0, -1, -30, np.nan]) +@pytest.mark.parametrize("convbl", [BOARDER, LODGER, NEITHER]) +def test_no_positive_amount_means_no_rent(convbl, cvpay): + boarder, lodger = frs_boarder_and_lodger_rent(make_person([(1_001, convbl, cvpay)])) + assert boarder.tolist() == [0] + assert lodger.tolist() == [0] + + +def test_adult_and_child_rows_sharing_index_labels(): + # create_frs stacks the adult and child tables and fills the gaps with + # zero, so index labels repeat and child rows carry CVPAY 0 and CONVBL 0. + adults = make_person([(1_001, NOT_ASKED, 0), (1_002, BOARDER, 70)]) + children = pd.DataFrame({"person_id": [1_003]}) + person = pd.concat([adults, children]).sort_index(kind="stable").fillna(0) + assert person.index.tolist() == [0, 0, 1] + boarder, lodger = frs_boarder_and_lodger_rent(person) + np.testing.assert_allclose(boarder, [0, 0, 70 * WEEKS_IN_YEAR]) + assert lodger.tolist() == [0, 0, 0] + + +def random_tables(seed: int): + """Random households of one to four adults; the first is the HRP.""" + rng = np.random.default_rng(seed) + people, households = [], [] + for household_id in range(1, rng.integers(1, 30) + 1): + households.append((household_id, int(rng.integers(1, 9)), 0.0)) + for person in range(1, rng.integers(1, 5) + 1): + people.append( + ( + household_id, + household_id * 1_000 + person, + HRP if person == 1 else NOT_HRP, + float(rng.choice([0, rng.uniform(0, 2_000)])), + rng.choice([BOARDER, LODGER, NEITHER, NOT_ASKED, -1, np.nan]), + rng.choice([0, rng.uniform(0, 400), -1, np.nan]), + ) + ) + person = pd.DataFrame( + people, + columns=["household_id", "person_id", "hrpid", "royyr1", "convbl", "cvpay"], + ) + household = pd.DataFrame( + households, columns=["household_id", "tentyp2", "subrent"] + ).set_index("household_id") + return person, household + + +SEEDS = range(200) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_boarder_and_lodger_rent_conserves_reported_rent(seed): + # Every positive CVPAY is counted once, in exactly one of the two classes. + person, _ = random_tables(seed) + boarder, lodger = frs_boarder_and_lodger_rent(person) + assert (boarder >= 0).all() and (lodger >= 0).all() + assert not ((boarder > 0) & (lodger > 0)).any() + paid = person.cvpay.where(person.cvpay > 0, 0) + np.testing.assert_allclose(boarder + lodger, paid * WEEKS_IN_YEAR) + np.testing.assert_allclose( + boarder.sum(), paid[person.convbl == BOARDER].sum() * WEEKS_IN_YEAR + ) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_extra_rent_moves_only_the_person_who_pays_it(seed): + person, _ = random_tables(seed) + person["cvpay"] = person.cvpay.where(person.cvpay > 0, 0) + before = sum(frs_boarder_and_lodger_rent(person)) + row = seed % len(person) + person.loc[row, "cvpay"] += 10 + after = sum(frs_boarder_and_lodger_rent(person)) + change = np.zeros(len(person)) + change[row] = 10 * WEEKS_IN_YEAR + np.testing.assert_allclose(after - before, change) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_rent_paid_and_property_income_do_not_affect_each_other(seed): + person, household = random_tables(seed) + property_income = frs_property_income(person.fillna(0), household) + rent_paid = frs_boarder_and_lodger_rent(person) + np.testing.assert_array_equal( + frs_property_income( + person.fillna(0).assign(cvpay=0.0, convbl=NOT_ASKED), household + ), + property_income, + ) + other_property = person.assign(royyr1=person.royyr1 + 50) + for changed, original in zip( + frs_boarder_and_lodger_rent(other_property), rent_paid + ): + np.testing.assert_array_equal(changed, original) + + +RENT_COLUMNS = ["rent_paid_as_boarder", "rent_paid_as_lodger"] + + +@pytest.mark.parametrize( + "table", ["uprating_factors.csv", "uprating_growth_factors.csv"] +) +def test_rent_paid_is_uprated_like_sublet_income(table): + # policyengine-uk uprates both inputs with the per capita GDP index it + # uses for sublet_income, so their rows must match. + from policyengine_uk_data.storage import STORAGE_FOLDER + + factors = pd.read_csv(STORAGE_FOLDER / table).set_index("Variable") + for column in RENT_COLUMNS: + pd.testing.assert_series_equal( + factors.loc[column], factors.loc["sublet_income"], check_names=False + ) + + +def test_rent_paid_is_uprated_to_the_calibration_year(): + # The build materialises the survey-year dataset at the calibration year + # before calibrating; the rent paid must move with it. + from policyengine_uk.data import UKSingleYearDataset + + from policyengine_uk_data.storage import STORAGE_FOLDER + from policyengine_uk_data.utils.uprating import uprate_dataset + + person = pd.DataFrame( + { + "person_id": [1_001, 1_002], + "person_benunit_id": [101, 102], + "person_household_id": [1, 1], + "rent_paid_as_boarder": [0.0, 100 * WEEKS_IN_YEAR], + "rent_paid_as_lodger": [80 * WEEKS_IN_YEAR, 0.0], + } + ) + benunit = pd.DataFrame({"benunit_id": [101, 102]}) + household = pd.DataFrame({"household_id": [1], "household_weight": [1.0]}) + dataset = UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=2024 + ) + uprated = uprate_dataset(dataset, 2025) + factors = pd.read_csv(STORAGE_FOLDER / "uprating_factors.csv").set_index("Variable") + growth = factors.loc["sublet_income", "2025"] / factors.loc["sublet_income", "2024"] + assert growth > 1 + for column in RENT_COLUMNS: + np.testing.assert_allclose(uprated.person[column], person[column] * growth) diff --git a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py index 5f1acd85..dc52389f 100644 --- a/policyengine_uk_data/tests/test_legacy_benefit_proxies.py +++ b/policyengine_uk_data/tests/test_legacy_benefit_proxies.py @@ -1,9 +1,11 @@ import numpy as np import pandas as pd import policyengine_uk +import pytest import policyengine_uk_data.datasets.frs as frs_module from policyengine_uk_data.datasets.frs import ( + WEEKS_IN_YEAR, add_legacy_benefit_proxies, attach_legacy_benefit_proxies_from_frs_person, apply_legacy_benefit_proxies, @@ -373,7 +375,13 @@ def calculate(self, variable, year=None): raise KeyError(variable) -def test_create_frs_smoke_includes_legacy_proxy_columns(tmp_path, monkeypatch): +@pytest.mark.parametrize( + "convbl, cvpay, boarder_weekly, lodger_weekly", + [(0, 0, 0, 0), (1, 100, 100, 0), (2, 80, 0, 80)], +) +def test_create_frs_smoke_includes_legacy_proxy_columns( + tmp_path, monkeypatch, convbl, cvpay, boarder_weekly, lodger_weekly +): original_read_csv = frs_module.pd.read_csv def fake_read_csv(path, *args, **kwargs): @@ -421,7 +429,8 @@ def fake_read_csv(path, *args, **kwargs): "ademaamt": 0, "age": 30, "age80": 30, - "cvpay": 0, + "convbl": convbl, + "cvpay": cvpay, "educft": 0, "educqual": 0, "eduma": 0, @@ -561,3 +570,10 @@ def fake_read_csv(path, *args, **kwargs): ].iloc[0] assert dataset.person["education_grants"].iloc[0] == 100 assert dataset.person["disabled_students_allowance_eligible_expenses"].iloc[0] == 0 + # create_frs carries the rent this person pays the householder, by class. + assert dataset.person["rent_paid_as_boarder"].iloc[0] == pytest.approx( + boarder_weekly * WEEKS_IN_YEAR + ) + assert dataset.person["rent_paid_as_lodger"].iloc[0] == pytest.approx( + lodger_weekly * WEEKS_IN_YEAR + )