From 5e4d58bd0fdfe072c940e279903858ce669b4677 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:06:07 -0400 Subject: [PATCH 1/4] Mark SPI Marriage Allowance recipients and transferors for the statutory model MAIND == 1 marks the spouse who receives a Marriage Allowance transfer, so marriage_allowance stays on those records under policyengine-uk#1963, where it is the amount received and gives a Step 6 tax reduction. Transferors are not flagged, but their SPI personal allowance is the allowance less the transferable amount; set marriage_allowance_relinquished for them. PAS is not an input, so neither side is counted twice. Co-Authored-By: Claude Opus 5.5 --- .../spi-marriage-allowance-sides.fixed.md | 1 + policyengine_uk_data/datasets/spi.py | 72 +++++----- .../tests/test_spi_allowance_deductions.py | 2 + policyengine_uk_data/tests/test_spi_build.py | 124 ++++++++++++++++-- 4 files changed, 154 insertions(+), 45 deletions(-) create mode 100644 changelog.d/spi-marriage-allowance-sides.fixed.md diff --git a/changelog.d/spi-marriage-allowance-sides.fixed.md b/changelog.d/spi-marriage-allowance-sides.fixed.md new file mode 100644 index 000000000..79806260b --- /dev/null +++ b/changelog.d/spi-marriage-allowance-sides.fixed.md @@ -0,0 +1 @@ +SPI dataset: document that `MAIND` marks the Marriage Allowance recipient, whose `marriage_allowance` is the amount received under policyengine-uk's statutory treatment, and give transferors (SPI personal allowance cut by the transferable amount) `marriage_allowance_relinquished`. diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index e22be675c..20cef2fb2 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -43,42 +43,34 @@ } -def _get_marriage_allowance(fiscal_year: int) -> float: - """Return the maximum Marriage Allowance transfer for the given UK fiscal - year in £. This equals ``max`` × ``personal_allowance`` at the start of - the fiscal year (6 April), which is how HMRC publishes it. Falls back to - the pre-2021-22 hard value of £1,250 if `policyengine_uk` cannot be - imported (e.g., during unit tests that avoid the heavy import). +def _get_allowances(fiscal_year: int) -> tuple[float, float, float]: + """Return the personal allowance (ITA 2007 s. 35(1)), the income above + which it tapers, and the Marriage Allowance transferable amount for the + given UK fiscal year in £. + + The transferable amount is 10% of the personal allowance, rounded up to + a multiple of £10 (s. 55B(4)-(5)), as in policyengine-uk's + ``marriage_allowance_transferable_amount``. Falls back to the 2020-21 + values if `policyengine_uk` cannot be imported (e.g., during unit tests + that avoid the heavy import). """ try: from policyengine_uk.system import system except Exception: - return 1_250.0 + return 12_500.0, 100_000.0, 1_250.0 instant = f"{fiscal_year}-04-06" - pa = system.parameters.gov.hmrc.income_tax.allowances.personal_allowance.amount( - instant - ) - ma_cap_rate = ( - system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.max(instant) - ) - # HMRC rounds to the nearest £10 downward; use the explicit rounding param - # if it exists, otherwise leave the computed value as-is. + allowances = system.parameters.gov.hmrc.income_tax.allowances + pa = allowances.personal_allowance.amount(instant) + taper_threshold = allowances.personal_allowance.maximum_ANI(instant) + transferable = pa * allowances.marriage_allowance.max(instant) try: - rounding_increment = system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.rounding_increment( - instant - ) + increment = float(allowances.marriage_allowance.rounding_increment(instant)) except Exception: - rounding_increment = None - - value = pa * ma_cap_rate - if rounding_increment: - # HMRC rounds the cap UP to the nearest rounding increment - # (Income Tax Act 2007 s. 55B(5)); matches the formula in - # policyengine_uk.variables.gov.hmrc.income_tax.allowances.marriage_allowance. - increment = float(rounding_increment) - value = np.ceil(value / increment) * increment - return float(value) + increment = None + if increment: + transferable = np.ceil(transferable / increment) * increment + return float(pa), float(taper_threshold), float(transferable) def create_spi( @@ -165,10 +157,26 @@ def create_spi( person["other_deductions"] = df.MOTHDED + df.DEFICIEN person["married_couples_allowance"] = df.MCAS person["blind_persons_allowance"] = df.BPADUE - # Pull the Marriage Allowance cap from policyengine-uk parameters keyed - # on the fiscal year, rather than hardcoding 2020-21's £1,250 figure. - ma_cap = _get_marriage_allowance(fiscal_year) - person["marriage_allowance"] = np.where(df.MAIND == 1, ma_cap, 0) + # HMRC documents MAIND as "Marriage allowance claimant indicator" (1 = + # "Claimant") and PAS as "Personal allowance (includes 10% marriage + # allowance transfer if applicable)". The label does not say which spouse + # claims, but in the 2022-23 tape every MAIND == 1 record has PAS equal to + # the personal allowance plus the transferable amount, so it marks the + # spouse who receives the transfer. policyengine-uk's `marriage_allowance` + # is that received amount, and gives a tax reduction + # (`marriage_allowance_tax_reduction`), not extra allowance. + pa, taper_threshold, transferable = _get_allowances(fiscal_year) + person["marriage_allowance"] = np.where(df.MAIND == 1, transferable, 0) + # The tape does not flag the spouse who transfers, but their PAS is the + # personal allowance less the transferable amount (s. 55B(6)). Below the + # taper threshold nothing else gives that value. Each record is one + # person, so policyengine-uk cannot find the electing spouse itself; give + # the allowance up directly. PAS is not an input, so neither side of a + # transfer is counted twice. + transferor = ( + (df.MAIND == 0) & (df.PAS == pa - transferable) & (df.TI < taper_threshold) + ) + person["marriage_allowance_relinquished"] = np.where(transferor, transferable, 0) dataset = UKSingleYearDataset( person=person, diff --git a/policyengine_uk_data/tests/test_spi_allowance_deductions.py b/policyengine_uk_data/tests/test_spi_allowance_deductions.py index f635bb1f6..8b62c52d6 100644 --- a/policyengine_uk_data/tests/test_spi_allowance_deductions.py +++ b/policyengine_uk_data/tests/test_spi_allowance_deductions.py @@ -38,6 +38,8 @@ def test_spi_overrides_allowance_deductions_not_policy_parameters(tmp_path): "MCAS": 0, "BPADUE": 0, "MAIND": 0, + "PAS": 12_500, + "TI": 2_000, } spi_path = tmp_path / "spi.tab" pd.DataFrame([row]).to_csv(spi_path, sep="\t", index=False) diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index efb37f810..52214415c 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -66,14 +66,17 @@ "MCAS", "BPADUE", "MAIND", + "PAS", + "TI", ] -def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1)): +def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1), **columns): """Write a minimal SPI-shaped tab file for tests. The real SPI file has dozens of columns; the test only needs them to exist with sensible types so ``create_spi`` can build dataframes. + ``columns`` overrides any column with per-row values. """ n = len(gor_values) data = {col: np.zeros(n, dtype=float) for col in SPI_COLUMNS} @@ -82,6 +85,7 @@ def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1)): data["GORCODE"] = list(gor_values) data["MAIND"] = list(maind_values) data["AGERANGE"] = [1] * n # bucket (16, 25) + data.update({col: list(values) for col, values in columns.items()}) df = pd.DataFrame(data) df.to_csv(path, sep="\t", index=False) @@ -144,12 +148,10 @@ def test_create_spi_unknown_gorcode_does_not_silently_become_south_east( def test_create_spi_marriage_allowance_uses_fiscal_year_parameters(tmp_path): - """MA cap should follow the fiscal year's 10% × Personal Allowance rule. - - 2020-21 PA = £12,500 so MA cap = £1,250 (the historical hardcoded value). - 2021-22 onwards PA = £12,570 so MA cap = £1,257, rounded down to - increments per the rounding_increment parameter (HMRC publishes £1,260 - for 2025-26). + """The transferable amount follows the fiscal year: 10% of the personal + allowance, rounded up to a multiple of £10 (ITA 2007 s. 55B(4)-(5)). + That is £1,250 in 2020-21 (allowance £12,500) and £1,260 from 2021-22 + (allowance £12,570). """ from policyengine_uk_data.datasets.spi import create_spi @@ -158,17 +160,113 @@ def test_create_spi_marriage_allowance_uses_fiscal_year_parameters(tmp_path): ds_2020 = create_spi(tab, 2020, seed=0) marriage_2020 = ds_2020.person["marriage_allowance"].to_numpy() - # Expect eligible rows (MAIND == 1) to receive £1,250 and ineligible 0. + # MAIND == 1 rows receive the transferable amount; others receive nothing. assert (marriage_2020[[0, 2]] == 1_250).all() assert marriage_2020[1] == 0 ds_2025 = create_spi(tab, 2025, seed=0) marriage_2025 = ds_2025.person["marriage_allowance"].to_numpy() - # Post-2020, PA is £12,570 so the cap is £1,257 before rounding; the - # published HMRC value is £1,260 (rounding to nearest £10). Accept - # either, but require it's NOT the stale 2020-21 £1,250 figure. - assert marriage_2025[0] != 1_250 - assert marriage_2025[0] >= 1_250 # PA has only risen since 2020 + assert (marriage_2025[[0, 2]] == 1_260).all() + assert marriage_2025[1] == 0 + + +def test_create_spi_marriage_allowance_sides(tmp_path): + """MAIND == 1 marks the recipient, who gets the transferable amount as + `marriage_allowance`. The tape has no transferor flag, but a transferor's + PAS is the personal allowance less the transferable amount, so that + record gives the amount up through `marriage_allowance_relinquished`. + A tapered allowance that happens to equal that value is not a transfer, + and composite records (MAIND == -1) are neither side. + """ + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + _write_fake_spi( + tab, + gor_values=(7, 7, 7, 7, 7), + maind_values=(1, 0, 0, 0, -1), + # recipient, transferor, no transfer, tapered, composite + PAS=(13_830, 11_310, 12_570, 11_310, 0), + TI=(30_000, 12_000, 30_000, 102_520, 600_000), + ) + + person = create_spi(tab, 2022, seed=0).person + + assert person["marriage_allowance"].tolist() == [1_260, 0, 0, 0, 0] + assert person["marriage_allowance_relinquished"].tolist() == [0, 1_260, 0, 0, 0] + # PAS already includes the transfer on both sides; it must stay out of + # the dataset so the model does not count the transfer twice. + assert "personal_allowance" not in person + + +def test_create_spi_marriage_allowance_invariants(tmp_path): + """Over every combination of MAIND, PAS and income: each side is 0 or + the transferable amount, no record is both sides, only MAIND == 1 + receives, and no one receiving or relinquishing lacks the matching + tape evidence. + """ + from itertools import product + + from policyengine_uk_data.datasets.spi import create_spi + + pa, transferable = 12_570, 1_260 + cases = list( + product( + (-1, 0, 1), + (0, pa - transferable, pa, pa + transferable, 5_000), + (8_000, 99_999, 100_000, 150_000), + ) + ) + maind, pas, ti = zip(*cases) + tab = tmp_path / "spi.tab" + _write_fake_spi( + tab, gor_values=[7] * len(cases), maind_values=maind, PAS=pas, TI=ti + ) + + person = create_spi(tab, 2022, seed=0).person + received = person["marriage_allowance"].to_numpy() + relinquished = person["marriage_allowance_relinquished"].to_numpy() + + assert set(received) <= {0, transferable} + assert set(relinquished) <= {0, transferable} + assert not ((received > 0) & (relinquished > 0)).any() + assert ((received > 0) == (np.array(maind) == 1)).all() + assert (np.array(pas)[relinquished > 0] == pa - transferable).all() + assert (np.array(ti)[relinquished > 0] < 100_000).all() + assert "personal_allowance" not in person + + +def test_spi_marriage_allowance_matches_statute_in_policyengine_uk(tmp_path): + """Run SPI-shaped records through policyengine-uk for 2022-23. + + The recipient's tax falls by the basic rate, or for a Scottish taxpayer + the Scottish basic rate, times the transferable amount (ITA 2007 s. + 55B(1), (3)): 20% x £1,260 = £252 in both cases, including a Scottish + intermediate-rate payer. The recipient's own allowance is unchanged. The + transferor's allowance falls by £1,260 (s. 55B(6)). + """ + from policyengine_uk import Microsimulation + from policyengine_uk_data.datasets.spi import create_spi + + tab = tmp_path / "spi.tab" + _write_fake_spi( + tab, + # London recipient, London no transfer, Scottish recipient, + # Scottish no transfer, London transferor + gor_values=(7, 7, 11, 11, 7), + maind_values=(1, 0, 1, 0, 0), + PAY=(30_000, 30_000, 30_000, 30_000, 12_000), + PAS=(13_830, 12_570, 13_830, 12_570, 11_310), + TI=(30_000, 30_000, 30_000, 30_000, 12_000), + ) + sim = Microsimulation(dataset=create_spi(tab, 2022, seed=0)) + tax = sim.calculate("income_tax", 2022).values + allowance = sim.calculate("personal_allowance", 2022).values + + assert tax[1] - tax[0] == pytest.approx(252) + assert tax[3] - tax[2] == pytest.approx(252) + assert allowance.tolist() == [12_570, 12_570, 12_570, 12_570, 11_310] + assert tax[4] == pytest.approx(0.2 * (12_000 - 11_310)) def test_current_spi_release_metadata_points_to_2022_23(): From 61754cab3b544ee98012c84c7e8ecc91fa43f0eb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:09:20 -0400 Subject: [PATCH 2/4] Skip the SPI statute test until policyengine-uk has the statutory Marriage Allowance The locked policyengine-uk predates PolicyEngine/policyengine-uk#1963. A lock bump changes the dataset build, so it belongs in its own release. The test runs as soon as the installed model defines marriage_allowance_relinquished. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/tests/test_spi_build.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index 52214415c..f5f320a02 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -246,8 +246,15 @@ def test_spi_marriage_allowance_matches_statute_in_policyengine_uk(tmp_path): transferor's allowance falls by £1,260 (s. 55B(6)). """ from policyengine_uk import Microsimulation + from policyengine_uk.system import system from policyengine_uk_data.datasets.spi import create_spi + if "marriage_allowance_relinquished" not in system.variables: + pytest.skip( + "needs a policyengine-uk release with the statutory Marriage " + "Allowance (PolicyEngine/policyengine-uk#1963)" + ) + tab = tmp_path / "spi.tab" _write_fake_spi( tab, From ba07c78552d7b2fac9718f43be875cfb45ea55ff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:15:33 -0400 Subject: [PATCH 3/4] Address review: fail loudly without policyengine-uk, pin MAIND == 0 for transferors The 2020-21 fallback in _get_allowances would have given a 2022-23 tape the wrong transferable amount and found no transferors, without an error. The module already needs policyengine-uk. The invariant grid now fails if a record other than MAIND == 0 relinquishes. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/datasets/spi.py | 17 ++++------------- policyengine_uk_data/tests/test_spi_build.py | 5 +++-- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/policyengine_uk_data/datasets/spi.py b/policyengine_uk_data/datasets/spi.py index 20cef2fb2..b04de2ad2 100644 --- a/policyengine_uk_data/datasets/spi.py +++ b/policyengine_uk_data/datasets/spi.py @@ -50,26 +50,17 @@ def _get_allowances(fiscal_year: int) -> tuple[float, float, float]: The transferable amount is 10% of the personal allowance, rounded up to a multiple of £10 (s. 55B(4)-(5)), as in policyengine-uk's - ``marriage_allowance_transferable_amount``. Falls back to the 2020-21 - values if `policyengine_uk` cannot be imported (e.g., during unit tests - that avoid the heavy import). + ``marriage_allowance_transferable_amount``. """ - try: - from policyengine_uk.system import system - except Exception: - return 12_500.0, 100_000.0, 1_250.0 + from policyengine_uk.system import system instant = f"{fiscal_year}-04-06" allowances = system.parameters.gov.hmrc.income_tax.allowances pa = allowances.personal_allowance.amount(instant) taper_threshold = allowances.personal_allowance.maximum_ANI(instant) + increment = allowances.marriage_allowance.rounding_increment(instant) transferable = pa * allowances.marriage_allowance.max(instant) - try: - increment = float(allowances.marriage_allowance.rounding_increment(instant)) - except Exception: - increment = None - if increment: - transferable = np.ceil(transferable / increment) * increment + transferable = np.ceil(transferable / increment) * increment return float(pa), float(taper_threshold), float(transferable) diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index f5f320a02..c43c2f4dd 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -202,8 +202,8 @@ def test_create_spi_marriage_allowance_sides(tmp_path): def test_create_spi_marriage_allowance_invariants(tmp_path): """Over every combination of MAIND, PAS and income: each side is 0 or the transferable amount, no record is both sides, only MAIND == 1 - receives, and no one receiving or relinquishing lacks the matching - tape evidence. + receives, and only an unflagged record with the cut allowance and income + under the taper threshold relinquishes. """ from itertools import product @@ -231,6 +231,7 @@ def test_create_spi_marriage_allowance_invariants(tmp_path): assert set(relinquished) <= {0, transferable} assert not ((received > 0) & (relinquished > 0)).any() assert ((received > 0) == (np.array(maind) == 1)).all() + assert (np.array(maind)[relinquished > 0] == 0).all() assert (np.array(pas)[relinquished > 0] == pa - transferable).all() assert (np.array(ti)[relinquished > 0] < 100_000).all() assert "personal_allowance" not in person From 8ec96d8329c7ab830cf1131b66aee8f5f219667e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 23:25:37 -0400 Subject: [PATCH 4/4] Assert that exactly the qualifying SPI records relinquish allowance Round 2 review: the grid only checked that relinquishers qualify, so a narrower transferor rule passed every test. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk_data/tests/test_spi_build.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/policyengine_uk_data/tests/test_spi_build.py b/policyengine_uk_data/tests/test_spi_build.py index c43c2f4dd..77b76d110 100644 --- a/policyengine_uk_data/tests/test_spi_build.py +++ b/policyengine_uk_data/tests/test_spi_build.py @@ -231,9 +231,12 @@ def test_create_spi_marriage_allowance_invariants(tmp_path): assert set(relinquished) <= {0, transferable} assert not ((received > 0) & (relinquished > 0)).any() assert ((received > 0) == (np.array(maind) == 1)).all() - assert (np.array(maind)[relinquished > 0] == 0).all() - assert (np.array(pas)[relinquished > 0] == pa - transferable).all() - assert (np.array(ti)[relinquished > 0] < 100_000).all() + qualifies = ( + (np.array(maind) == 0) + & (np.array(pas) == pa - transferable) + & (np.array(ti) < 100_000) + ) + assert ((relinquished > 0) == qualifies).all() assert "personal_allowance" not in person