diff --git a/changelog.d/spi-marriage-allowance-sides.fixed.md b/changelog.d/spi-marriage-allowance-sides.fixed.md new file mode 100644 index 00000000..79806260 --- /dev/null +++ b/changelog.d/spi-marriage-allowance-sides.fixed.md @@ -0,0 +1 @@ +SPI dataset: document that `MAIND` marks the Marriage Allowance recipient, whose `marriage_allowance` is the amount received under policyengine-uk's statutory treatment, and give transferors (SPI personal allowance cut by the transferable amount) `marriage_allowance_relinquished`. diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index e22be675..b04de2ad 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -43,42 +43,25 @@ } -def _get_marriage_allowance(fiscal_year: int) -> float: - """Return the maximum Marriage Allowance transfer for the given UK fiscal - year in £. This equals ``max`` × ``personal_allowance`` at the start of - the fiscal year (6 April), which is how HMRC publishes it. Falls back to - the pre-2021-22 hard value of £1,250 if `policyengine_uk` cannot be - imported (e.g., during unit tests that avoid the heavy import). +def _get_allowances(fiscal_year: int) -> tuple[float, float, float]: + """Return the personal allowance (ITA 2007 s. 35(1)), the income above + which it tapers, and the Marriage Allowance transferable amount for the + given UK fiscal year in £. + + The transferable amount is 10% of the personal allowance, rounded up to + a multiple of £10 (s. 55B(4)-(5)), as in policyengine-uk's + ``marriage_allowance_transferable_amount``. """ - try: - from policyengine_uk.system import system - except Exception: - return 1_250.0 + from policyengine_uk.system import system instant = f"{fiscal_year}-04-06" - pa = system.parameters.gov.hmrc.income_tax.allowances.personal_allowance.amount( - instant - ) - ma_cap_rate = ( - system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.max(instant) - ) - # HMRC rounds to the nearest £10 downward; use the explicit rounding param - # if it exists, otherwise leave the computed value as-is. - try: - rounding_increment = system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.rounding_increment( - instant - ) - except Exception: - rounding_increment = None - - value = pa * ma_cap_rate - if rounding_increment: - # HMRC rounds the cap UP to the nearest rounding increment - # (Income Tax Act 2007 s. 55B(5)); matches the formula in - # policyengine_uk.variables.gov.hmrc.income_tax.allowances.marriage_allowance. - increment = float(rounding_increment) - value = np.ceil(value / increment) * increment - return float(value) + allowances = system.parameters.gov.hmrc.income_tax.allowances + pa = allowances.personal_allowance.amount(instant) + taper_threshold = allowances.personal_allowance.maximum_ANI(instant) + increment = allowances.marriage_allowance.rounding_increment(instant) + transferable = pa * allowances.marriage_allowance.max(instant) + transferable = np.ceil(transferable / increment) * increment + return float(pa), float(taper_threshold), float(transferable) def create_spi( @@ -165,10 +148,26 @@ def create_spi( person["other_deductions"] = df.MOTHDED + df.DEFICIEN person["married_couples_allowance"] = df.MCAS person["blind_persons_allowance"] = df.BPADUE - # Pull the Marriage Allowance cap from policyengine-uk parameters keyed - # on the fiscal year, rather than hardcoding 2020-21's £1,250 figure. - ma_cap = _get_marriage_allowance(fiscal_year) - person["marriage_allowance"] = np.where(df.MAIND == 1, ma_cap, 0) + # HMRC documents MAIND as "Marriage allowance claimant indicator" (1 = + # "Claimant") and PAS as "Personal allowance (includes 10% marriage + # allowance transfer if applicable)". The label does not say which spouse + # claims, but in the 2022-23 tape every MAIND == 1 record has PAS equal to + # the personal allowance plus the transferable amount, so it marks the + # spouse who receives the transfer. policyengine-uk's `marriage_allowance` + # is that received amount, and gives a tax reduction + # (`marriage_allowance_tax_reduction`), not extra allowance. + pa, taper_threshold, transferable = _get_allowances(fiscal_year) + person["marriage_allowance"] = np.where(df.MAIND == 1, transferable, 0) + # The tape does not flag the spouse who transfers, but their PAS is the + # personal allowance less the transferable amount (s. 55B(6)). Below the + # taper threshold nothing else gives that value. Each record is one + # person, so policyengine-uk cannot find the electing spouse itself; give + # the allowance up directly. PAS is not an input, so neither side of a + # transfer is counted twice. + transferor = ( + (df.MAIND == 0) & (df.PAS == pa - transferable) & (df.TI < taper_threshold) + ) + person["marriage_allowance_relinquished"] = np.where(transferor, transferable, 0) dataset = UKSingleYearDataset( person=person, diff --git a/policyengine_uk_data/tests/test_spi_allowance_deductions.py b/policyengine_uk_data/tests/test_spi_allowance_deductions.py index f635bb1f..8b62c52d 100644 --- a/policyengine_uk_data/tests/test_spi_allowance_deductions.py +++ b/policyengine_uk_data/tests/test_spi_allowance_deductions.py @@ -38,6 +38,8 @@ def test_spi_overrides_allowance_deductions_not_policy_parameters(tmp_path): "MCAS": 0, "BPADUE": 0, "MAIND": 0, + "PAS": 12_500, + "TI": 2_000, } spi_path = tmp_path / "spi.tab" pd.DataFrame([row]).to_csv(spi_path, sep="\t", index=False) diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index efb37f81..77b76d11 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -66,14 +66,17 @@ "MCAS", "BPADUE", "MAIND", + "PAS", + "TI", ] -def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1)): +def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1), **columns): """Write a minimal SPI-shaped tab file for tests. The real SPI file has dozens of columns; the test only needs them to exist with sensible types so ``create_spi`` can build dataframes. + ``columns`` overrides any column with per-row values. """ n = len(gor_values) data = {col: np.zeros(n, dtype=float) for col in SPI_COLUMNS} @@ -82,6 +85,7 @@ def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1)): data["GORCODE"] = list(gor_values) data["MAIND"] = list(maind_values) data["AGERANGE"] = [1] * n # bucket (16, 25) + data.update({col: list(values) for col, values in columns.items()}) df = pd.DataFrame(data) df.to_csv(path, sep="\t", index=False) @@ -144,12 +148,10 @@ def test_create_spi_unknown_gorcode_does_not_silently_become_south_east( def test_create_spi_marriage_allowance_uses_fiscal_year_parameters(tmp_path): - """MA cap should follow the fiscal year's 10% × Personal Allowance rule. - - 2020-21 PA = £12,500 so MA cap = £1,250 (the historical hardcoded value). - 2021-22 onwards PA = £12,570 so MA cap = £1,257, rounded down to - increments per the rounding_increment parameter (HMRC publishes £1,260 - for 2025-26). + """The transferable amount follows the fiscal year: 10% of the personal + allowance, rounded up to a multiple of £10 (ITA 2007 s. 55B(4)-(5)). + That is £1,250 in 2020-21 (allowance £12,500) and £1,260 from 2021-22 + (allowance £12,570). """ from policyengine_uk_data.datasets.spi import create_spi @@ -158,17 +160,124 @@ def test_create_spi_marriage_allowance_uses_fiscal_year_parameters(tmp_path): ds_2020 = create_spi(tab, 2020, seed=0) marriage_2020 = ds_2020.person["marriage_allowance"].to_numpy() - # Expect eligible rows (MAIND == 1) to receive £1,250 and ineligible 0. + # MAIND == 1 rows receive the transferable amount; others receive nothing. assert (marriage_2020[[0, 2]] == 1_250).all() assert marriage_2020[1] == 0 ds_2025 = create_spi(tab, 2025, seed=0) marriage_2025 = ds_2025.person["marriage_allowance"].to_numpy() - # Post-2020, PA is £12,570 so the cap is £1,257 before rounding; the - # published HMRC value is £1,260 (rounding to nearest £10). Accept - # either, but require it's NOT the stale 2020-21 £1,250 figure. - assert marriage_2025[0] != 1_250 - assert marriage_2025[0] >= 1_250 # PA has only risen since 2020 + assert (marriage_2025[[0, 2]] == 1_260).all() + assert marriage_2025[1] == 0 + + +def test_create_spi_marriage_allowance_sides(tmp_path): + """MAIND == 1 marks the recipient, who gets the transferable amount as + `marriage_allowance`. The tape has no transferor flag, but a transferor's + PAS is the personal allowance less the transferable amount, so that + record gives the amount up through `marriage_allowance_relinquished`. + A tapered allowance that happens to equal that value is not a transfer, + and composite records (MAIND == -1) are neither side. + """ + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + _write_fake_spi( + tab, + gor_values=(7, 7, 7, 7, 7), + maind_values=(1, 0, 0, 0, -1), + # recipient, transferor, no transfer, tapered, composite + PAS=(13_830, 11_310, 12_570, 11_310, 0), + TI=(30_000, 12_000, 30_000, 102_520, 600_000), + ) + + person = create_spi(tab, 2022, seed=0).person + + assert person["marriage_allowance"].tolist() == [1_260, 0, 0, 0, 0] + assert person["marriage_allowance_relinquished"].tolist() == [0, 1_260, 0, 0, 0] + # PAS already includes the transfer on both sides; it must stay out of + # the dataset so the model does not count the transfer twice. + assert "personal_allowance" not in person + + +def test_create_spi_marriage_allowance_invariants(tmp_path): + """Over every combination of MAIND, PAS and income: each side is 0 or + the transferable amount, no record is both sides, only MAIND == 1 + receives, and only an unflagged record with the cut allowance and income + under the taper threshold relinquishes. + """ + from itertools import product + + from policyengine_uk_data.datasets.spi import create_spi + + pa, transferable = 12_570, 1_260 + cases = list( + product( + (-1, 0, 1), + (0, pa - transferable, pa, pa + transferable, 5_000), + (8_000, 99_999, 100_000, 150_000), + ) + ) + maind, pas, ti = zip(*cases) + tab = tmp_path / "spi.tab" + _write_fake_spi( + tab, gor_values=[7] * len(cases), maind_values=maind, PAS=pas, TI=ti + ) + + person = create_spi(tab, 2022, seed=0).person + received = person["marriage_allowance"].to_numpy() + relinquished = person["marriage_allowance_relinquished"].to_numpy() + + assert set(received) <= {0, transferable} + assert set(relinquished) <= {0, transferable} + assert not ((received > 0) & (relinquished > 0)).any() + assert ((received > 0) == (np.array(maind) == 1)).all() + qualifies = ( + (np.array(maind) == 0) + & (np.array(pas) == pa - transferable) + & (np.array(ti) < 100_000) + ) + assert ((relinquished > 0) == qualifies).all() + assert "personal_allowance" not in person + + +def test_spi_marriage_allowance_matches_statute_in_policyengine_uk(tmp_path): + """Run SPI-shaped records through policyengine-uk for 2022-23. + + The recipient's tax falls by the basic rate, or for a Scottish taxpayer + the Scottish basic rate, times the transferable amount (ITA 2007 s. + 55B(1), (3)): 20% x £1,260 = £252 in both cases, including a Scottish + intermediate-rate payer. The recipient's own allowance is unchanged. The + transferor's allowance falls by £1,260 (s. 55B(6)). + """ + from policyengine_uk import Microsimulation + from policyengine_uk.system import system + from policyengine_uk_data.datasets.spi import create_spi + + if "marriage_allowance_relinquished" not in system.variables: + pytest.skip( + "needs a policyengine-uk release with the statutory Marriage " + "Allowance (PolicyEngine/policyengine-uk#1963)" + ) + + tab = tmp_path / "spi.tab" + _write_fake_spi( + tab, + # London recipient, London no transfer, Scottish recipient, + # Scottish no transfer, London transferor + gor_values=(7, 7, 11, 11, 7), + maind_values=(1, 0, 1, 0, 0), + PAY=(30_000, 30_000, 30_000, 30_000, 12_000), + PAS=(13_830, 12_570, 13_830, 12_570, 11_310), + TI=(30_000, 30_000, 30_000, 30_000, 12_000), + ) + sim = Microsimulation(dataset=create_spi(tab, 2022, seed=0)) + tax = sim.calculate("income_tax", 2022).values + allowance = sim.calculate("personal_allowance", 2022).values + + assert tax[1] - tax[0] == pytest.approx(252) + assert tax[3] - tax[2] == pytest.approx(252) + assert allowance.tolist() == [12_570, 12_570, 12_570, 12_570, 11_310] + assert tax[4] == pytest.approx(0.2 * (12_000 - 11_310)) def test_current_spi_release_metadata_points_to_2022_23():