Calibrate SPI income bands to their projected values, and map 2023-24 outturns to 2023

changelog.d/spi-cgt-tax-year-start.fixed.md

@@ -0,0 +1 @@
+Map the HMRC Survey of Personal Incomes and capital gains tax 2023-24 outturns to PolicyEngine year 2023, the tax year they describe, instead of 2024. Return each SPI income band as one target that holds both the base year and the projected years, so calibration uses the projected values rather than the base-year outturn carried forward. Together these raise the 2025 SPI income targets by two years of uprating (employment income +8.6%) and the capital gains targets by one (+3.8%).

policyengine_uk_data/targets/sources/hmrc_cgt.py

@@ -29,17 +29,19 @@
 for the simulation year, so the target's meaning tracks the policy in
 force. ``_AEA_FALLBACK`` is used only if the parameter lookup fails.
 
-Year handling: HMRC's figures are the 2023-24 outturn, mapped to
-calendar 2024 following ``hmrc_spi._SPI_YEAR``. The amount is projected
-to 2029 with PolicyEngine's ``capital_gains`` uprating factors so the
-target constrains the projection years the calibration actually runs
-for; the taxpayer count is held flat, as the repo has no administrative
-basis for forecasting CGT taxpayer numbers and count uprating factors
-would be a nominal-income index applied to a headcount.
-
-Caveat: the 2023-24 outturn was realised under a £6,000 AEA, while the
-gate applied in later years uses the £3,000 AEA then in force. The
-targets are therefore approximate for the projection years; the
+Year handling: HMRC's figures are the 2023-24 outturn. PolicyEngine
+UK's year N is tax year N to N+1, so the outturn is the 2023 value, the
+year whose £6,000 AEA it was realised under (``hmrc_spi._SPI_YEAR``
+follows the same convention). The amount is projected to 2029 with
+PolicyEngine's ``capital_gains`` uprating factors so the target
+constrains the projection years the calibration actually runs for; the
+taxpayer count is held flat, as the repo has no administrative basis for
+forecasting CGT taxpayer numbers and count uprating factors would be a
+nominal-income index applied to a headcount.
+
+Caveat: from 2024 the gate uses the £3,000 AEA then in force, while the
+projected values carry the 2023-24 outturn realised under a £6,000 AEA.
+The targets are therefore approximate for the projection years; the
 alternative (a fixed £6,000 gate) would be wrong in a different and less
 transparent way, because it would not describe any year's actual policy.
 
@@ -55,9 +57,9 @@
 
 logger = logging.getLogger(__name__)
 
-# HMRC CGT statistics, tax year 2023-24 outturn, mapped to calendar 2024
-# following the hmrc_spi._SPI_YEAR convention.
-_CGT_BASE_YEAR = 2024
+# HMRC CGT statistics, tax year 2023-24 outturn. PolicyEngine UK's year N
+# is tax year N to N+1, so the outturn is the 2023 value.
+_CGT_BASE_YEAR = 2023
 _MAX_YEAR = 2029
 
 # Total chargeable gains of CGT taxpayers, 2023-24 (£65.9bn).

policyengine_uk_data/targets/sources/hmrc_spi.py

@@ -53,8 +53,11 @@
 ]
 _BAND_UPPER = _BAND_LOWER[1:] + [float("inf")]
 
-# SPI year: the ODS is for tax year 2023-24, mapped to calendar 2024
-_SPI_YEAR = 2024
+# SPI year: the ODS is for tax year 2023-24. PolicyEngine UK's year N is
+# tax year N to N+1 (policyengine-uk's convert_to_fiscal_year_parameters
+# reads each parameter at 30 April N), so the outturn belongs to 2023.
+# Later years come from incomes_projection.csv.
+_SPI_YEAR = 2023
 
 # HMRC Property Rental Income Statistics show ~1.9x more property income
 # than the SPI (£46.68bn vs £24.5bn for 2020-21), because SPI only covers
@@ -229,8 +232,13 @@ def get_income_band_table(include_aggregate: bool = True) -> pd.DataFrame:
 def get_targets() -> list[Target]:
     """Build income-band targets from the live HMRC SPI ODS.
 
-    Also reads incomes_projection.csv if available, which contains
-    projected future year data generated by the microsimulation.
+    Also reads incomes_projection.csv if available, which projects the
+    same bands to later years with PolicyEngine's uprating indices. Each
+    band is returned as one target holding the base year and every
+    projected year: the registry keeps only the first target of a name,
+    and calibration reads targets without a year filter, so a band split
+    across two targets would lose its projected years and be calibrated
+    to the base-year outturn carried forward.
     """
     config = load_config()
     ref = config["hmrc"]["spi_collated"]
@@ -291,7 +299,18 @@ def get_targets() -> list[Target]:
     if proj_path.exists():
         targets.extend(_read_projection_csv(proj_path, ref))
 
-    return targets
+    return _merge_by_name(targets)
+
+
+def _merge_by_name(targets: list[Target]) -> list[Target]:
+    """Combine targets that share a name into one, merging their years."""
+    merged: dict[str, Target] = {}
+    for t in targets:
+        if t.name in merged:
+            merged[t.name].values.update(t.values)
+        else:
+            merged[t.name] = t
+    return list(merged.values())
 
 
 def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]:
@@ -355,12 +374,4 @@ def _read_projection_csv(csv_path: Path, ref: str) -> list[Target]:
                         )
                     )
 
-    # Merge targets with the same name across years
-    merged: dict[str, Target] = {}
-    for t in targets:
-        if t.name in merged:
-            merged[t.name].values.update(t.values)
-        else:
-            merged[t.name] = t
-
-    return list(merged.values())
+    return _merge_by_name(targets)

policyengine_uk_data/tests/test_hmrc_cgt_targets.py

@@ -89,6 +89,20 @@ def test_aea_tracks_the_year():
     assert compute_cgt_taxpayers(_dummy_ctx(**args), target, 2023)[0] == 0.0
 
 
+def test_base_year_is_the_tax_year_the_outturn_was_realised_in():
+    """PolicyEngine UK's year N is tax year N to N+1, so the 2023-24 outturn
+    is the 2023 value. The base-year gate must then be 2023-24's £6,000 AEA,
+    read from the real parameter tree; mapping the outturn to 2024 would gate
+    it at 2024-25's £3,000."""
+    from policyengine_uk.system import system
+
+    from policyengine_uk_data.targets.sources.hmrc_cgt import _annual_exempt_amount
+
+    ctx = SimpleNamespace(sim=SimpleNamespace(tax_benefit_system=system))
+    assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR) == 6_000
+    assert _annual_exempt_amount(ctx, _CGT_BASE_YEAR + 1) == 3_000
+
+
 def test_targets_load_with_expected_base_year_values():
     targets = {t.name: t for t in get_targets()}
     # Aggregate targets plus the per-size-band targets (tested in
@@ -296,7 +310,7 @@ def test_registry_keeps_custom_compute_for_the_calibration_years():
 
 
 def test_registry_year_filter_excludes_years_before_the_outturn():
-    """The targets describe a 2023-24 outturn mapped to calendar 2024,
+    """The targets describe the 2023-24 outturn, PolicyEngine year 2023,
     so they must not appear for earlier years."""
     from policyengine_uk_data.targets import get_all_targets
 

policyengine_uk_data/tests/test_income_projection.py

@@ -5,9 +5,12 @@
 to have been generated and will be skipped otherwise.
 """
 
+import re
+
 import pytest
 import pandas as pd
 from policyengine_uk_data.storage import STORAGE_FOLDER
+from policyengine_uk_data.targets.sources._common import load_config
 from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR
 
 PROJECTION_PATH = STORAGE_FOLDER / "incomes_projection.csv"
@@ -143,3 +146,81 @@ def test_projection_keeps_top_open_ended_band(projections):
     ]
     assert len(top_band) == 1
     assert top_band.iloc[0]["dividend_income_amount"] > 0
+
+
+def test_projection_is_the_spi_table_uprated_with_the_committed_table(
+    projections, base_targets
+):
+    """Regenerate incomes_projection.csv whenever uprating_factors.csv changes."""
+    from policyengine_uk_data.utils.incomes_projection import project_income_table
+
+    pd.testing.assert_frame_equal(project_income_table(base_targets), projections)
+
+
+def test_spi_year_is_the_start_of_the_configured_tax_year():
+    """The SPI ODS name ends with its tax year (..._2324.ods is 2023-24).
+
+    PolicyEngine UK's year N is tax year N to N+1 (each parameter takes its
+    value at 30 April N), so a 2023-24 outturn is the 2023 value. Mapping it
+    to the end year would treat it as 2024-25 and understate every later
+    year by a year's growth. A refreshed URL without a matching _SPI_YEAR
+    fails here.
+    """
+    url = load_config()["hmrc"]["spi_collated"]
+    match = re.search(r"_(\d{2})(\d{2})\.ods$", url)
+    assert match, url
+    start, end = int(match[1]), int(match[2])
+    assert end == (start + 1) % 100, url
+    assert _SPI_YEAR == 2000 + start
+
+
+def test_calibration_reads_the_projected_spi_targets(
+    projections, base_targets, monkeypatch
+):
+    """Calibration (build_loss_matrix) fetches targets without a year filter,
+    and the registry keeps only the first target of each name. Every SPI band
+    must therefore be one target carrying its outturn and every projected
+    year; if the projected years sit on a second target of the same name they
+    are dropped, and the outturn is carried forward unuprated instead.
+
+    The official table is served from the committed incomes.csv, so the
+    outturn targets are always present and the test needs no download.
+    """
+    from policyengine_uk_data.targets import get_all_targets
+    from policyengine_uk_data.targets.build_loss_matrix import _resolve_value
+    from policyengine_uk_data.targets.sources import hmrc_spi
+
+    def committed_table(include_aggregate: bool = True) -> pd.DataFrame:
+        if include_aggregate:
+            return base_targets
+        return _without_aggregate(base_targets).reset_index(drop=True)
+
+    monkeypatch.setattr(hmrc_spi, "get_income_band_table", committed_table)
+
+    names = [t.name for t in hmrc_spi.get_targets()]
+    assert len(names) == len(set(names))
+
+    spi = {t.name: t for t in get_all_targets() if t.source == "hmrc_spi"}
+    expected = _without_aggregate(projections[projections["year"] >= _SPI_YEAR])
+    checked = 0
+    for row in expected.itertuples():
+        label = hmrc_spi._format_band_label(
+            row.total_income_lower_bound, row.total_income_upper_bound
+        )
+        for variable in hmrc_spi.INCOME_VARIABLES:
+            amount = getattr(row, f"{variable}_amount")
+            if variable == "property_income":
+                amount *= hmrc_spi._PROPERTY_INCOME_SCALE
+            amount_target = spi[f"hmrc/{variable}_income_band_{label}"]
+            count_target = spi[f"hmrc/{variable}_count_income_band_{label}"]
+            assert _resolve_value(amount_target, row.year) == pytest.approx(amount)
+            assert _resolve_value(count_target, row.year) == pytest.approx(
+                getattr(row, f"{variable}_count")
+            )
+            checked += 2
+    assert checked == 2 * len(hmrc_spi.INCOME_VARIABLES) * len(expected)
+
+    years = set(projections["year"])
+    assert _SPI_YEAR in years
+    for target in spi.values():
+        assert set(target.values) == years, target.name

policyengine_uk_data/tests/test_property_income_targets.py

@@ -5,6 +5,7 @@
 """
 
 from policyengine_uk_data.targets import get_all_targets
+from policyengine_uk_data.targets.sources.hmrc_spi import _SPI_YEAR
 
 
 def test_property_income_targets_scaled():
@@ -13,7 +14,7 @@ def test_property_income_targets_scaled():
     Raw SPI 2023-24 total is scaled up to better match HMRC rental
     income statistics, which cover more landlords than SPI.
     """
-    base_year = 2024
+    base_year = _SPI_YEAR
     targets = get_all_targets(year=base_year)
     total = sum(
         t.values[base_year]