diff --git a/changelog.d/salary-sacrifice-needs-pay.fixed.md b/changelog.d/salary-sacrifice-needs-pay.fixed.md new file mode 100644 index 00000000..7f968229 --- /dev/null +++ b/changelog.d/salary-sacrifice-needs-pay.fixed.md @@ -0,0 +1,2 @@ +- Limit every salary sacrifice amount, reported or imputed, to the person's pay, and train the salary sacrifice QRF only on SALSAC respondents with pay. Stage 1 had drawn amounts for people without pay from the few SALSAC respondents who report a sacrifice but have no recorded pay, so about a million people with no pay carried salary sacrifice, and from 2029 policyengine-uk returned the excess over the £2,000 cap to their employment income. +- Stage 2 of the salary sacrifice imputation now takes the employee pension contribution it converts to salary sacrifice out of the donor's pay (at most half the pay). It had left pay unchanged, so converted donors lost the income tax relief on that contribution. diff --git a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py index 35800b33..83a57b92 100644 --- a/policyengine_uk_data/datasets/imputations/salary_sacrifice.py +++ b/policyengine_uk_data/datasets/imputations/salary_sacrifice.py @@ -8,9 +8,18 @@ not asked. 2. Headcount-targeted imputation: converts a fraction of pension - contributors without SS into below-cap (≤£2,000) SS users, moving - employee pension contributions to salary sacrifice. Targets the - OBR/ASHE estimate of ~4.3mn below-cap SS users. + contributors without SS into SS users, moving employee pension + contributions to salary sacrifice and taking the amount moved out of + their pay, so the calibrator has enough records to reach the OBR + headcounts (7.7mn users: 4.3mn at or below £2,000, 3.3mn above). + +A salary sacrifice is pay given up for an employer pension contribution, +so every amount, reported or imputed, is limited to the person's pay: +nobody without pay sacrifices anything. Without that limit, stage 1 drew +amounts for people with no pay from the few SALSAC respondents who have a +job but no recorded pay, and from 2029 policyengine-uk returns the excess +over the £2,000 cap to employment income, giving those people pay they do +not have. Exact monetary totals (~£24bn SS contributions) and final headcount calibration happen via weight optimisation in a subsequent step. @@ -23,6 +32,10 @@ from policyengine_uk.data import UKSingleYearDataset from policyengine_uk import Microsimulation +# Trained only on SALSAC respondents with pay; the name changed with that +# training set so a forest cached by an older build is not reused. +MODEL_FILE = "salary_sacrifice_paid_respondents.pkl" + PREDICTORS = [ "age", "employment_income", @@ -33,14 +46,37 @@ ] +def limit_salary_sacrifice_to_pay( + salary_sacrifice: np.ndarray, employment_income: np.ndarray +) -> np.ndarray: + """Limit salary sacrifice amounts to the pay they are sacrificed from. + + A person without pay sacrifices nothing. The upper bound is an + imputation bound, not a legal one (an employee may sacrifice, for + example, a whole bonus, as long as cash pay stays above the National + Minimum Wage): no FRS 2024-25 respondent who reports a sacrifice and + has pay reports more than the pay they still receive, so imputed + amounts are held to the same bound. + + Args: + salary_sacrifice: Annual salary sacrifice pension contributions. + employment_income: Annual employment income after the sacrifice. + + Returns: + Amounts between zero and the person's pay (zero without pay). + """ + return np.clip(salary_sacrifice, 0, np.maximum(employment_income, 0)) + + def save_salary_sacrifice_model(): """ Train and save salary sacrifice imputation model using FRS data. - Uses FRS respondents who were asked about salary sacrifice (SALSAC field) - as training data. The model learns to predict the SS pension amount - directly - non-participants have 0, participants have their reported - SPNAMT value. + Uses FRS respondents with pay who were asked about salary sacrifice + (SALSAC field) as training data. The model learns to predict the SS + pension amount directly - non-participants have 0, participants have + their reported SPNAMT value. Respondents without pay are left out: only + predictions for people with pay are kept. Returns: Trained QRF model for salary sacrifice imputation. @@ -71,12 +107,15 @@ def save_salary_sacrifice_model(): ) ss_asked = dataset.person.salary_sacrifice_asked.values - # Build training DataFrame with only those who were asked + # Build training DataFrame with only those who were asked and have pay # This includes both participants (with amounts) and non-participants (0) - training_mask = ss_asked == 1 + training_mask = (ss_asked == 1) & (employment_income > 0) if training_mask.sum() == 0: - raise ValueError("No training data found - no respondents were asked SALSAC.") + raise ValueError( + "No training data found - no SALSAC respondents with positive " + "employment income." + ) train_df = pd.DataFrame( { @@ -89,7 +128,7 @@ def save_salary_sacrifice_model(): # Train QRF model model = QRF() model.fit(train_df[PREDICTORS], train_df[IMPUTATIONS]) - model.save(STORAGE_FOLDER / "salary_sacrifice.pkl") + model.save(STORAGE_FOLDER / MODEL_FILE) return model @@ -106,7 +145,7 @@ def create_salary_sacrifice_model(overwrite_existing: bool = False): """ from policyengine_uk_data.utils.qrf import QRF - model_path = STORAGE_FOLDER / "salary_sacrifice.pkl" + model_path = STORAGE_FOLDER / MODEL_FILE if model_path.exists() and not overwrite_existing: return QRF(file_path=model_path) return save_salary_sacrifice_model() @@ -119,9 +158,12 @@ def impute_salary_sacrifice( Impute salary sacrifice pension amounts for FRS non-respondents. Stage 1: QRF predicts SS amounts for respondents not asked SALSAC. - Stage 2: Converts a fraction of pension contributors to below-cap - SS users, targeting ~4.3mn (OBR/ASHE). Moves employee pension - contributions to salary sacrifice to keep total pension consistent. + Every amount, reported or predicted, is then limited to the person's + pay (see ``limit_salary_sacrifice_to_pay``). + Stage 2: Converts a fraction of pension contributors to SS users, + for the OBR headcount targets. Moves employee pension + contributions to salary sacrifice to keep total pension consistent, + and takes the amount moved out of pay (at most half the pay). Args: dataset: PolicyEngine UK dataset with salary_sacrifice_asked @@ -133,9 +175,13 @@ def impute_salary_sacrifice( dataset = dataset.copy() sim = Microsimulation(dataset=dataset) - # Get variables needed for imputation + # Get variables needed for imputation. The predictors come from the + # simulation, as in training; the limits use the stored pay, which is in + # the same year as the stored salary sacrifice (the simulation may + # calculate a later year and uprate pay but not salary sacrifice). age = sim.calculate("age").values employment_income = sim.calculate("employment_income").values + pay = dataset.person["employment_income"].values.copy() current_ss = dataset.person.pension_contributions_via_salary_sacrifice.values # Get indicator for who was asked @@ -166,23 +212,28 @@ def impute_salary_sacrifice( # For those not asked, use the imputed values final_ss = np.where( ss_asked == 1, - current_ss, # Keep reported values exactly + current_ss, # Keep reported values imputed_ss, # Use imputed for non-respondents ) + # A sacrifice comes out of pay. This also covers SPI-synthetic rows + # copied from a SALSAC respondent, whose kept amounts come from the + # FRS-only imputation rather than the survey. + final_ss = limit_salary_sacrifice_to_pay(final_ss, pay) # Stage 2: Headcount-targeted imputation for SS users. # ASHE data shows many more SS users than the FRS captures due to # self-reporting bias in auto-enrolment. Impute additional SS users # from pension contributors to create enough records for calibration # to hit OBR headcount targets (7.7mn total, 3.3mn above 2k, - # 4.3mn below 2k). Donors keep their full employee pension amount - # so those above 2k become above-cap records and the rest below-cap. + # 4.3mn below 2k). Donors move their employee pension amount (at + # most half their pay), so those above 2k become above-cap records + # and the rest below-cap. person_weight = sim.calculate("person_weight").values employee_pension = dataset.person["employee_pension_contributions"].values.copy() has_ss = final_ss > 0 # Donor pool: employed pension contributors not already SS users - is_donor = (employee_pension > 0) & ~has_ss & (employment_income > 0) + is_donor = (employee_pension > 0) & ~has_ss & (pay > 0) # Create enough SS records for the calibrator to work with. # Target ~70% of the 7.7mn total so the calibrator can gently @@ -198,17 +249,25 @@ def impute_salary_sacrifice( rng = np.random.default_rng(seed=2024) newly_imputed = is_donor & (rng.random(len(final_ss)) < imputation_rate) - # Move full employee pension to SS so the above/below - # 2k split reflects the natural pension distribution - ss_new = employee_pension.copy() + # Move full employee pension to SS so the above/below 2k + # split reflects the natural pension distribution. The FRS + # pay these contributions were deducted from is before the + # sacrifice, so the sacrifice comes out of it: pay falls by + # the amount moved, which keeps pay plus sacrifice unchanged. At most + # half the pay moves, so the sacrifice never exceeds the pay + # that remains. + ss_new = limit_salary_sacrifice_to_pay(employee_pension, pay / 2) final_ss = np.where(newly_imputed, ss_new, final_ss) - # Reduce employee pension correspondingly + # Reduce employee pension and pay correspondingly dataset.person["employee_pension_contributions"] = np.where( newly_imputed, - 0.0, + employee_pension - ss_new, employee_pension, ) + dataset.person["employment_income"] = np.where( + newly_imputed, pay - ss_new, pay + ) dataset.person["pension_contributions_via_salary_sacrifice"] = final_ss diff --git a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py index 9ed48abf..41d5c784 100644 --- a/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py +++ b/policyengine_uk_data/tests/test_pension_contributions_via_salary_sacrifice.py @@ -1,3 +1,7 @@ +import numpy as np +import pytest + + def test_pension_contributions_via_salary_sacrifice(baseline): """Test that pension_contributions_via_salary_sacrifice loads and has reasonable values.""" values = baseline.calculate( @@ -20,3 +24,50 @@ def test_pension_contributions_via_salary_sacrifice(baseline): assert total < total_employment, ( f"Salary sacrifice contributions ({total / 1e9:.1f}B) cannot exceed total employment income ({total_employment / 1e9:.1f}B)" ) + + +def test_salary_sacrifice_needs_pay(baseline): + """A salary sacrifice comes out of pay: none without pay, none above it.""" + pay = baseline.calculate("employment_income_before_lsr", period=2025).values + ss = baseline.calculate( + "pension_contributions_via_salary_sacrifice", period=2025 + ).values + weight = baseline.calculate("person_weight", period=2025).values + + without_pay = (ss > 0) & (pay <= 0) + assert not without_pay.any(), ( + f"{without_pay.sum()} people ({weight[without_pay].sum() / 1e3:.0f}k " + "weighted) have salary sacrifice but no pay" + ) + # Both columns uprate by the same index, so only rounding can separate them. + above_pay = ss > pay * (1 + 1e-9) + 1e-6 + assert not above_pay.any(), ( + f"{above_pay.sum()} people sacrifice more than their pay" + ) + + +def test_salary_sacrifice_cap_gives_nobody_pay_they_do_not_have(baseline): + """From 2029 the excess over the cap returns to pay, so it needs pay.""" + pay = baseline.calculate("employment_income_before_lsr", period=2029).values + employment_income = baseline.calculate("employment_income", period=2029).values + phantom = (pay <= 0) & (employment_income > 0) + assert not phantom.any(), ( + f"{phantom.sum()} people without pay have employment income in 2029" + ) + + +def test_model_pay_limit_never_binds_on_the_dataset(baseline): + """policyengine-uk limits salary sacrifice to pay as the imputation does.""" + limited = "pension_contributions_via_salary_sacrifice_from_pay" + if limited not in baseline.tax_benefit_system.variables: + pytest.skip(f"policyengine-uk has no {limited} variable") + for period in (2025, 2029): + ss = baseline.calculate( + "pension_contributions_via_salary_sacrifice", period=period + ).values + from_pay = baseline.calculate(limited, period=period).values + differing = ~np.isclose(from_pay, ss, rtol=1e-6, atol=1e-2) + assert not differing.any(), ( + f"policyengine-uk limits the salary sacrifice of {differing.sum()} " + f"people in {period}" + ) diff --git a/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py new file mode 100644 index 00000000..574b1fb8 --- /dev/null +++ b/policyengine_uk_data/tests/test_salary_sacrifice_imputation.py @@ -0,0 +1,248 @@ +"""Salary sacrifice imputation keeps every sacrifice within the person's pay. + +Invariants of ``limit_salary_sacrifice_to_pay``, for every input: + +- the result is between zero and the person's pay, so zero without pay; +- an amount already within those bounds is unchanged; +- applying it twice changes nothing; +- it is non-decreasing in the sacrifice and in the pay. + +``impute_salary_sacrifice`` applies it to every record, reported or +imputed. Stage 2 moves employee pension contributions to salary sacrifice +and takes the amount moved out of pay, so for every donor the pension total +and pay plus salary sacrifice are both unchanged, and the sacrifice never +exceeds the pay that remains. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk_data.datasets.imputations import ( + salary_sacrifice as salary_sacrifice_module, +) +from policyengine_uk_data.datasets.imputations.salary_sacrifice import ( + limit_salary_sacrifice_to_pay, +) + +amounts = st.floats(min_value=-1e5, max_value=1e7, allow_nan=False) +pairs = st.lists(st.tuples(amounts, amounts), min_size=1, max_size=40) + + +def _split(rows): + ss, pay = np.array(rows, dtype=float).T + return ss, pay + + +@given(pairs) +def test_result_is_between_zero_and_pay(rows): + ss, pay = _split(rows) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.all(limited >= 0) + assert np.all(limited <= np.maximum(pay, 0)) + assert np.all(limited[pay <= 0] == 0) + + +@given(pairs) +def test_amounts_within_pay_are_unchanged(rows): + ss, pay = _split(rows) + within = (ss >= 0) & (ss <= pay) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.array_equal(limited[within], ss[within]) + + +@given(pairs) +def test_limit_is_idempotent(rows): + ss, pay = _split(rows) + once = limit_salary_sacrifice_to_pay(ss, pay) + assert np.array_equal(limit_salary_sacrifice_to_pay(once, pay), once) + + +@given(pairs, st.floats(min_value=0, max_value=1e6), st.floats(0, 1e6)) +def test_limit_is_monotone_in_sacrifice_and_pay(rows, more_ss, more_pay): + ss, pay = _split(rows) + limited = limit_salary_sacrifice_to_pay(ss, pay) + assert np.all(limit_salary_sacrifice_to_pay(ss + more_ss, pay) >= limited) + assert np.all(limit_salary_sacrifice_to_pay(ss, pay + more_pay) >= limited) + + +# The build imputes on the survey-year dataset (2024) while the simulation +# calculates a later default year, uprating pay but not salary sacrifice, so +# the wiring is checked in both years. +YEARS = pytest.mark.parametrize("year", [2024, 2025]) + + +class _StubModel: + """Stands in for the trained QRF: predicts a fixed amount per row.""" + + def __init__(self, predictions): + self.predictions = np.asarray(predictions, dtype=float) + + def predict(self, X): + assert len(X) == len(self.predictions) + return pd.DataFrame( + {"pension_contributions_via_salary_sacrifice": self.predictions} + ) + + +def _dataset(pay, asked, reported, employee_pension, year): + from policyengine_uk.data import UKSingleYearDataset + + n = len(pay) + person = pd.DataFrame( + { + "person_id": np.arange(n), + "person_benunit_id": np.arange(n), + "person_household_id": np.arange(n), + "age": 40, + "employment_income": np.asarray(pay, dtype=float), + "salary_sacrifice_asked": np.asarray(asked, dtype=int), + "pension_contributions_via_salary_sacrifice": np.asarray( + reported, dtype=float + ), + "employee_pension_contributions": np.asarray(employee_pension, dtype=float), + } + ) + benunit = pd.DataFrame({"benunit_id": np.arange(n)}) + household = pd.DataFrame( + { + "household_id": np.arange(n), + "household_weight": 1.0, + "region": "LONDON", + "tenure_type": "OWNED_OUTRIGHT", + "council_tax": 0.0, + "rent": 0.0, + } + ) + return UKSingleYearDataset( + person=person, benunit=benunit, household=household, fiscal_year=year + ) + + +def _impute(monkeypatch, predictions, year=2024, **columns): + monkeypatch.setattr( + salary_sacrifice_module, + "create_salary_sacrifice_model", + lambda: _StubModel(predictions), + ) + dataset = _dataset(year=year, **columns) + before = dataset.person.copy() + after = salary_sacrifice_module.impute_salary_sacrifice(dataset).person + return before, after + + +def _assert_invariants(before, after): + pay_before = before.employment_income.values + pay = after.employment_income.values + ss = after.pension_contributions_via_salary_sacrifice.values + assert np.all(ss >= 0) + assert np.all(ss <= np.maximum(pay, 0)) + assert np.all(ss[pay <= 0] == 0) + # Stage 2 moves employee pension to salary sacrifice out of pay: whoever + # lost employee pension gained exactly that much salary sacrifice and + # lost exactly that much pay. Nobody else's pay changes. + pension_before = before.employee_pension_contributions.values + pension_after = after.employee_pension_contributions.values + moved = pension_after != pension_before + assert np.all(pension_after <= pension_before) + assert np.allclose(pension_after[moved] + ss[moved], pension_before[moved]) + assert np.allclose(pay[moved] + ss[moved], pay_before[moved]) + assert np.array_equal(pay[~moved], pay_before[~moved]) + + +@YEARS +def test_nobody_without_pay_keeps_a_sacrifice(monkeypatch, year): + pay = [0, 0, -500, 30_000, 30_000, 8_000, 0] + before, after = _impute( + monkeypatch, + year=year, + # The model predicts a large sacrifice for everyone. + predictions=[25_000] * 7, + pay=pay, + asked=[0, 1, 0, 0, 1, 0, 1], + reported=[0, 6_000, 0, 0, 1_200, 0, 0], + employee_pension=[0] * 7, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + # Unpaid: imputed (0, 2) and reported (1) amounts are all dropped. + assert ss[[0, 1, 2, 6]].tolist() == [0, 0, 0, 0] + # Paid and not asked: the prediction, limited to pay. + assert ss[3] == 25_000 + assert ss[5] == 8_000 + # Paid and asked: the reported amount is kept. + assert ss[4] == 1_200 + _assert_invariants(before, after) + + +@YEARS +def test_stage_two_takes_the_sacrifice_out_of_pay(monkeypatch, year): + n = 40 + before, after = _impute( + monkeypatch, + year=year, + predictions=[0] * n, + pay=[30_000] * n, + asked=[0] * n, + reported=[0] * n, + employee_pension=[1_500] * n, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + moved = ss > 0 + assert moved.any() and not moved.all() + assert np.all(ss[moved] == 1_500) + assert np.all(after.employee_pension_contributions.values[moved] == 0) + assert np.all(after.employment_income.values[moved] == 28_500) + assert np.all(after.employment_income.values[~moved] == 30_000) + _assert_invariants(before, after) + + +@YEARS +def test_stage_two_moves_at_most_half_the_pay(monkeypatch, year): + n = 40 + # Every donor's employee pension exceeds half their pay, so each one + # moved sacrifices half and keeps the rest as employee pension. + before, after = _impute( + monkeypatch, + year=year, + predictions=[0] * n, + pay=[1_000] * n, + asked=[0] * n, + reported=[0] * n, + employee_pension=[1_500] * n, + ) + ss = after.pension_contributions_via_salary_sacrifice.values + moved = ss > 0 + assert moved.any() + assert np.all(ss[moved] == 500) + assert np.all(after.employee_pension_contributions.values[moved] == 1_000) + assert np.all(after.employment_income.values[moved] == 500) + _assert_invariants(before, after) + + +row = st.tuples( + st.sampled_from([-1_000.0, 0.0, 500.0, 15_000.0, 60_000.0]), # pay + st.integers(0, 1), # asked SALSAC + st.sampled_from([0.0, 800.0, 6_000.0, 70_000.0]), # reported amount + st.sampled_from([0.0, 300.0, 3_000.0, 90_000.0]), # employee pension + st.sampled_from([-100.0, 0.0, 1_500.0, 27_000.0]), # model prediction +) + + +@settings(max_examples=10, deadline=None) +@given(st.lists(row, min_size=1, max_size=12)) +def test_imputation_keeps_every_sacrifice_within_pay(rows): + pay, asked, reported, employee_pension, predictions = map(list, zip(*rows)) + with pytest.MonkeyPatch.context() as monkeypatch: + before, after = _impute( + monkeypatch, + predictions=predictions, + pay=pay, + asked=asked, + reported=reported, + employee_pension=employee_pension, + ) + _assert_invariants(before, after)