SPI: keep Marriage Allowance on recipients (MAIND) and give transferors their cut allowance

changelog.d/spi-marriage-allowance-sides.fixed.md

@@ -0,0 +1 @@
+SPI dataset: document that `MAIND` marks the Marriage Allowance recipient, whose `marriage_allowance` is the amount received under policyengine-uk's statutory treatment, and give transferors (SPI personal allowance cut by the transferable amount) `marriage_allowance_relinquished`.

policyengine_uk_data/datasets/spi.py

@@ -43,42 +43,25 @@
 }
 
 
-def _get_marriage_allowance(fiscal_year: int) -> float:
-    """Return the maximum Marriage Allowance transfer for the given UK fiscal
-    year in £. This equals ``max`` × ``personal_allowance`` at the start of
-    the fiscal year (6 April), which is how HMRC publishes it. Falls back to
-    the pre-2021-22 hard value of £1,250 if `policyengine_uk` cannot be
-    imported (e.g., during unit tests that avoid the heavy import).
+def _get_allowances(fiscal_year: int) -> tuple[float, float, float]:
+    """Return the personal allowance (ITA 2007 s. 35(1)), the income above
+    which it tapers, and the Marriage Allowance transferable amount for the
+    given UK fiscal year in £.
+
+    The transferable amount is 10% of the personal allowance, rounded up to
+    a multiple of £10 (s. 55B(4)-(5)), as in policyengine-uk's
+    ``marriage_allowance_transferable_amount``.
     """
-    try:
-        from policyengine_uk.system import system
-    except Exception:
-        return 1_250.0
+    from policyengine_uk.system import system
 
     instant = f"{fiscal_year}-04-06"
-    pa = system.parameters.gov.hmrc.income_tax.allowances.personal_allowance.amount(
-        instant
-    )
-    ma_cap_rate = (
-        system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.max(instant)
-    )
-    # HMRC rounds to the nearest £10 downward; use the explicit rounding param
-    # if it exists, otherwise leave the computed value as-is.
-    try:
-        rounding_increment = system.parameters.gov.hmrc.income_tax.allowances.marriage_allowance.rounding_increment(
-            instant
-        )
-    except Exception:
-        rounding_increment = None
-
-    value = pa * ma_cap_rate
-    if rounding_increment:
-        # HMRC rounds the cap UP to the nearest rounding increment
-        # (Income Tax Act 2007 s. 55B(5)); matches the formula in
-        # policyengine_uk.variables.gov.hmrc.income_tax.allowances.marriage_allowance.
-        increment = float(rounding_increment)
-        value = np.ceil(value / increment) * increment
-    return float(value)
+    allowances = system.parameters.gov.hmrc.income_tax.allowances
+    pa = allowances.personal_allowance.amount(instant)
+    taper_threshold = allowances.personal_allowance.maximum_ANI(instant)
+    increment = allowances.marriage_allowance.rounding_increment(instant)
+    transferable = pa * allowances.marriage_allowance.max(instant)
+    transferable = np.ceil(transferable / increment) * increment
+    return float(pa), float(taper_threshold), float(transferable)
 
 
 def create_spi(
@@ -165,10 +148,26 @@ def create_spi(
     person["other_deductions"] = df.MOTHDED + df.DEFICIEN
     person["married_couples_allowance"] = df.MCAS
     person["blind_persons_allowance"] = df.BPADUE
-    # Pull the Marriage Allowance cap from policyengine-uk parameters keyed
-    # on the fiscal year, rather than hardcoding 2020-21's £1,250 figure.
-    ma_cap = _get_marriage_allowance(fiscal_year)
-    person["marriage_allowance"] = np.where(df.MAIND == 1, ma_cap, 0)
+    # HMRC documents MAIND as "Marriage allowance claimant indicator" (1 =
+    # "Claimant") and PAS as "Personal allowance (includes 10% marriage
+    # allowance transfer if applicable)". The label does not say which spouse
+    # claims, but in the 2022-23 tape every MAIND == 1 record has PAS equal to
+    # the personal allowance plus the transferable amount, so it marks the
+    # spouse who receives the transfer. policyengine-uk's `marriage_allowance`
+    # is that received amount, and gives a tax reduction
+    # (`marriage_allowance_tax_reduction`), not extra allowance.
+    pa, taper_threshold, transferable = _get_allowances(fiscal_year)
+    person["marriage_allowance"] = np.where(df.MAIND == 1, transferable, 0)
+    # The tape does not flag the spouse who transfers, but their PAS is the
+    # personal allowance less the transferable amount (s. 55B(6)). Below the
+    # taper threshold nothing else gives that value. Each record is one
+    # person, so policyengine-uk cannot find the electing spouse itself; give
+    # the allowance up directly. PAS is not an input, so neither side of a
+    # transfer is counted twice.
+    transferor = (
+        (df.MAIND == 0) & (df.PAS == pa - transferable) & (df.TI < taper_threshold)
+    )
+    person["marriage_allowance_relinquished"] = np.where(transferor, transferable, 0)
 
     dataset = UKSingleYearDataset(
         person=person,

policyengine_uk_data/tests/test_spi_allowance_deductions.py

@@ -38,6 +38,8 @@ def test_spi_overrides_allowance_deductions_not_policy_parameters(tmp_path):
         "MCAS": 0,
         "BPADUE": 0,
         "MAIND": 0,
+        "PAS": 12_500,
+        "TI": 2_000,
     }
     spi_path = tmp_path / "spi.tab"
     pd.DataFrame([row]).to_csv(spi_path, sep="\t", index=False)

policyengine_uk_data/tests/test_spi_build.py

@@ -66,14 +66,17 @@
     "MCAS",
     "BPADUE",
     "MAIND",
+    "PAS",
+    "TI",
 ]
 
 
-def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1)):
+def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1), **columns):
     """Write a minimal SPI-shaped tab file for tests.
 
     The real SPI file has dozens of columns; the test only needs them to
     exist with sensible types so ``create_spi`` can build dataframes.
+    ``columns`` overrides any column with per-row values.
     """
     n = len(gor_values)
     data = {col: np.zeros(n, dtype=float) for col in SPI_COLUMNS}
@@ -82,6 +85,7 @@ def _write_fake_spi(path, gor_values=(1, 2, 3), maind_values=(1, 0, 1)):
     data["GORCODE"] = list(gor_values)
     data["MAIND"] = list(maind_values)
     data["AGERANGE"] = [1] * n  # bucket (16, 25)
+    data.update({col: list(values) for col, values in columns.items()})
     df = pd.DataFrame(data)
     df.to_csv(path, sep="\t", index=False)
 
@@ -144,12 +148,10 @@ def test_create_spi_unknown_gorcode_does_not_silently_become_south_east(
 
 
 def test_create_spi_marriage_allowance_uses_fiscal_year_parameters(tmp_path):
-    """MA cap should follow the fiscal year's 10% × Personal Allowance rule.
-
-    2020-21 PA = £12,500 so MA cap = £1,250 (the historical hardcoded value).
-    2021-22 onwards PA = £12,570 so MA cap = £1,257, rounded down to
-    increments per the rounding_increment parameter (HMRC publishes £1,260
-    for 2025-26).
+    """The transferable amount follows the fiscal year: 10% of the personal
+    allowance, rounded up to a multiple of £10 (ITA 2007 s. 55B(4)-(5)).
+    That is £1,250 in 2020-21 (allowance £12,500) and £1,260 from 2021-22
+    (allowance £12,570).
     """
     from policyengine_uk_data.datasets.spi import create_spi
 
@@ -158,17 +160,124 @@ def test_create_spi_marriage_allowance_uses_fiscal_year_parameters(tmp_path):
 
     ds_2020 = create_spi(tab, 2020, seed=0)
     marriage_2020 = ds_2020.person["marriage_allowance"].to_numpy()
-    # Expect eligible rows (MAIND == 1) to receive £1,250 and ineligible 0.
+    # MAIND == 1 rows receive the transferable amount; others receive nothing.
     assert (marriage_2020[[0, 2]] == 1_250).all()
     assert marriage_2020[1] == 0
 
     ds_2025 = create_spi(tab, 2025, seed=0)
     marriage_2025 = ds_2025.person["marriage_allowance"].to_numpy()
-    # Post-2020, PA is £12,570 so the cap is £1,257 before rounding; the
-    # published HMRC value is £1,260 (rounding to nearest £10). Accept
-    # either, but require it's NOT the stale 2020-21 £1,250 figure.
-    assert marriage_2025[0] != 1_250
-    assert marriage_2025[0] >= 1_250  # PA has only risen since 2020
+    assert (marriage_2025[[0, 2]] == 1_260).all()
+    assert marriage_2025[1] == 0
+
+
+def test_create_spi_marriage_allowance_sides(tmp_path):
+    """MAIND == 1 marks the recipient, who gets the transferable amount as
+    `marriage_allowance`. The tape has no transferor flag, but a transferor's
+    PAS is the personal allowance less the transferable amount, so that
+    record gives the amount up through `marriage_allowance_relinquished`.
+    A tapered allowance that happens to equal that value is not a transfer,
+    and composite records (MAIND == -1) are neither side.
+    """
+    from policyengine_uk_data.datasets.spi import create_spi
+
+    tab = tmp_path / "spi.tab"
+    _write_fake_spi(
+        tab,
+        gor_values=(7, 7, 7, 7, 7),
+        maind_values=(1, 0, 0, 0, -1),
+        # recipient, transferor, no transfer, tapered, composite
+        PAS=(13_830, 11_310, 12_570, 11_310, 0),
+        TI=(30_000, 12_000, 30_000, 102_520, 600_000),
+    )
+
+    person = create_spi(tab, 2022, seed=0).person
+
+    assert person["marriage_allowance"].tolist() == [1_260, 0, 0, 0, 0]
+    assert person["marriage_allowance_relinquished"].tolist() == [0, 1_260, 0, 0, 0]
+    # PAS already includes the transfer on both sides; it must stay out of
+    # the dataset so the model does not count the transfer twice.
+    assert "personal_allowance" not in person
+
+
+def test_create_spi_marriage_allowance_invariants(tmp_path):
+    """Over every combination of MAIND, PAS and income: each side is 0 or
+    the transferable amount, no record is both sides, only MAIND == 1
+    receives, and only an unflagged record with the cut allowance and income
+    under the taper threshold relinquishes.
+    """
+    from itertools import product
+
+    from policyengine_uk_data.datasets.spi import create_spi
+
+    pa, transferable = 12_570, 1_260
+    cases = list(
+        product(
+            (-1, 0, 1),
+            (0, pa - transferable, pa, pa + transferable, 5_000),
+            (8_000, 99_999, 100_000, 150_000),
+        )
+    )
+    maind, pas, ti = zip(*cases)
+    tab = tmp_path / "spi.tab"
+    _write_fake_spi(
+        tab, gor_values=[7] * len(cases), maind_values=maind, PAS=pas, TI=ti
+    )
+
+    person = create_spi(tab, 2022, seed=0).person
+    received = person["marriage_allowance"].to_numpy()
+    relinquished = person["marriage_allowance_relinquished"].to_numpy()
+
+    assert set(received) <= {0, transferable}
+    assert set(relinquished) <= {0, transferable}
+    assert not ((received > 0) & (relinquished > 0)).any()
+    assert ((received > 0) == (np.array(maind) == 1)).all()
+    qualifies = (
+        (np.array(maind) == 0)
+        & (np.array(pas) == pa - transferable)
+        & (np.array(ti) < 100_000)
+    )
+    assert ((relinquished > 0) == qualifies).all()
+    assert "personal_allowance" not in person
+
+
+def test_spi_marriage_allowance_matches_statute_in_policyengine_uk(tmp_path):
+    """Run SPI-shaped records through policyengine-uk for 2022-23.
+
+    The recipient's tax falls by the basic rate, or for a Scottish taxpayer
+    the Scottish basic rate, times the transferable amount (ITA 2007 s.
+    55B(1), (3)): 20% x £1,260 = £252 in both cases, including a Scottish
+    intermediate-rate payer. The recipient's own allowance is unchanged. The
+    transferor's allowance falls by £1,260 (s. 55B(6)).
+    """
+    from policyengine_uk import Microsimulation
+    from policyengine_uk.system import system
+    from policyengine_uk_data.datasets.spi import create_spi
+
+    if "marriage_allowance_relinquished" not in system.variables:
+        pytest.skip(
+            "needs a policyengine-uk release with the statutory Marriage "
+            "Allowance (PolicyEngine/policyengine-uk#1963)"
+        )
+
+    tab = tmp_path / "spi.tab"
+    _write_fake_spi(
+        tab,
+        # London recipient, London no transfer, Scottish recipient,
+        # Scottish no transfer, London transferor
+        gor_values=(7, 7, 11, 11, 7),
+        maind_values=(1, 0, 1, 0, 0),
+        PAY=(30_000, 30_000, 30_000, 30_000, 12_000),
+        PAS=(13_830, 12_570, 13_830, 12_570, 11_310),
+        TI=(30_000, 30_000, 30_000, 30_000, 12_000),
+    )
+    sim = Microsimulation(dataset=create_spi(tab, 2022, seed=0))
+    tax = sim.calculate("income_tax", 2022).values
+    allowance = sim.calculate("personal_allowance", 2022).values
+
+    assert tax[1] - tax[0] == pytest.approx(252)
+    assert tax[3] - tax[2] == pytest.approx(252)
+    assert allowance.tolist() == [12_570, 12_570, 12_570, 12_570, 11_310]
+    assert tax[4] == pytest.approx(0.2 * (12_000 - 11_310))
 
 
 def test_current_spi_release_metadata_points_to_2022_23():