Calibrate households' SDLT to the OBR total less corporate SDLT

changelog.d/sdlt-target-excludes-corporate.fixed.md

@@ -0,0 +1 @@
+Calibrate households' SDLT (`stamp_duty_land_tax`) to the OBR's SDLT forecast less policyengine-uk's corporate SDLT statistic for the same year, instead of the whole OBR line. policyengine-uk adds SDLT paid by companies and other non-natural persons separately, as `corporate_sdlt`, so calibration counted that share twice: on the 1.58.0 dataset with PolicyEngine/policyengine-uk#2322, 2026-27 SDLT came to £23.29bn (£17.31bn households, £5.98bn corporate) against the OBR's £18.15bn. The 2026-27 household target becomes £12.17bn. Full builds now need a policyengine-uk with #2322, which projects the corporate statistic with the OBR forecast, and stop at the start with an earlier one, which holds it at £3.13bn in every year.

policyengine_uk_data/datasets/create_datasets.py

@@ -58,7 +58,8 @@ def _materialize_base_year_dataset(dataset, frs_release, uprate_dataset):
 def main():
     """Create enhanced FRS dataset with rich progress tracking."""
     try:
-        assert_local_build_environment()
+        # TESTING builds (PR CI) skip the full-build checks.
+        assert_local_build_environment(full_build=os.environ.get("TESTING", "0") != "1")
 
         from policyengine_uk.data import UKSingleYearDataset
         from policyengine_uk_data.datasets.disability_benefits import (

policyengine_uk_data/targets/sources/obr.py

@@ -193,6 +193,65 @@ def _find_row(ws, label: str, col: str = "B", max_row: int = 80) -> int:
     raise ValueError(f"Row '{label}' not found in sheet")
 
 
+def corporate_sdlt_receipts(years=None) -> dict[int, float]:
+    """policyengine-uk's SDLT receipts from non-natural persons, by year.
+
+    These are the statistics `corporate_sdlt` allocates to households: each
+    household's `shareholding` divides its corporate-sector wealth by the
+    weighted total, so the weighted sum of `corporate_sdlt` is this statistic
+    whatever the weights. Read from the installed policyengine-uk, which
+    projects it from PolicyEngine/policyengine-uk#2322 on. Years are
+    PolicyEngine years (2025 is 2025-26); the default is the OBR's years.
+    """
+    from policyengine_uk.system import system
+
+    statistics = system.parameters.gov.hmrc.stamp_duty.statistics
+    return {
+        year: float(
+            statistics.residential.corporate.revenue(f"{year}-01-01")
+            + statistics.non_residential.corporate.revenue(f"{year}-01-01")
+        )
+        for year in (_FY_COL_TO_YEAR.values() if years is None else years)
+    }
+
+
+def corporate_sdlt_is_projected(corporate: dict[int, float]) -> bool:
+    """Whether the corporate SDLT statistic changes over the years given.
+
+    policyengine-uk before PolicyEngine/policyengine-uk#2322 held it at its
+    2020-21 value, £3.13bn, in every later year.
+    """
+    return len(set(corporate.values())) > 1
+
+
+def household_sdlt(obr_sdlt: dict[int, float]) -> dict[int, float]:
+    """The OBR's SDLT receipts less policyengine-uk's corporate SDLT, by year.
+
+    The OBR's SDLT line covers every buyer. policyengine-uk counts receipts
+    from non-natural persons (companies, partnerships, collective investment
+    schemes) separately, as `corporate_sdlt`, so targeting the household
+    variable `stamp_duty_land_tax` at the whole line counted them twice (#564).
+    """
+    corporate = corporate_sdlt_receipts(obr_sdlt)
+    if len(corporate) > 1 and not corporate_sdlt_is_projected(corporate):
+        logger.warning(
+            "OBR receipts: policyengine-uk holds corporate SDLT at £%.2fbn in "
+            "every year from %d to %d, so it predates "
+            "PolicyEngine/policyengine-uk#2322 and the household SDLT target "
+            "overstates households' share. Full builds refuse it.",
+            next(iter(corporate.values())) / 1e9,
+            min(corporate),
+            max(corporate),
+        )
+    household = {year: total - corporate[year] for year, total in obr_sdlt.items()}
+    negative = sorted(year for year, value in household.items() if value < 0)
+    if negative:
+        raise ValueError(
+            f"policyengine-uk's corporate SDLT exceeds the OBR's SDLT in {negative}"
+        )
+    return household
+
+
 def _parse_receipts(wb: openpyxl.Workbook) -> list[Target]:
     """Parse tax receipts from the OBR EFO.
 
@@ -269,6 +328,8 @@ def read_39(ws, row_num: int) -> dict[int, float]:
         try:
             row_num = _find_row(ws39, label, col="B", max_row=80)
             values = read_39(ws39, row_num)
+            if values and name == "sdlt":
+                values = household_sdlt(values)
             if values:
                 recovered.append(label)
                 targets.append(
@@ -282,8 +343,8 @@ def read_39(ws, row_num: int) -> dict[int, float]:
                         forecast_vintage=vintage,
                     )
                 )
-        except ValueError:
-            logger.warning("OBR receipts: row '%s' not found", label)
+        except ValueError as e:
+            logger.warning("OBR receipts: row '%s' not used: %s", label, e)
 
     # Guard on targets actually produced, not on exceptions caught: a layout
     # change that shifts the value columns finds every row and still yields

policyengine_uk_data/tests/test_build_environment.py

@@ -29,3 +29,26 @@ def test_get_local_build_issues_accepts_supported_environment():
         )
         == []
     )
+
+
+def test_get_local_build_issues_requires_projected_corporate_sdlt():
+    """policyengine-uk before #2322 held corporate SDLT at £3.13bn."""
+    issues = get_local_build_issues(
+        python_version=(3, 13, 7),
+        variable_names={"num_vehicles"},
+        corporate_sdlt=dict.fromkeys(range(2024, 2031), 3.13e9),
+    )
+
+    assert len(issues) == 1
+    assert "PolicyEngine/policyengine-uk#2322" in issues[0]
+
+
+def test_get_local_build_issues_accepts_projected_corporate_sdlt():
+    assert (
+        get_local_build_issues(
+            python_version=(3, 13, 7),
+            variable_names={"num_vehicles"},
+            corporate_sdlt={2024: 4.49e9, 2025: 4.81e9, 2026: 5.98e9},
+        )
+        == []
+    )

policyengine_uk_data/tests/test_obr_receipts_sheet.py

@@ -102,8 +102,10 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook:
         row = 10 + offset
         ws[f"B{row}"] = label
         if populate:
+            # £100bn a year, above the corporate SDLT statistic the SDLT
+            # target subtracts.
             for col in ("D", "E", "F", "G", "H", "I", "J"):
-                ws[f"{col}{row}"] = 1.0
+                ws[f"{col}{row}"] = 100.0
     return wb
 
 

policyengine_uk_data/tests/test_obr_sdlt_target.py

@@ -0,0 +1,142 @@
+"""The OBR SDLT target leaves out the corporate SDLT policyengine-uk adds (#564).
+
+The OBR's SDLT line covers every buyer. policyengine-uk adds SDLT paid by
+non-natural persons as `corporate_sdlt` (its corporate statistic times each
+household's shareholding), so household `stamp_duty_land_tax` is targeted at
+the OBR line less that statistic.
+
+Invariants, for every year the OBR forecasts:
+- target + corporate statistic = OBR SDLT;
+- 0 <= target <= OBR SDLT whenever 0 <= statistic <= OBR SDLT, and a statistic
+  above the OBR figure is an error rather than a negative target;
+- the statistic subtracted is the one `corporate_sdlt` adds.
+"""
+
+import logging
+from unittest.mock import patch
+
+import pytest
+from hypothesis import given
+from hypothesis import strategies as st
+
+from policyengine_uk_data.targets.sources import obr
+from policyengine_uk_data.targets.sources._common import load_config
+
+YEARS = sorted(obr._FY_COL_TO_YEAR.values())
+
+
+@pytest.fixture(scope="module")
+def receipts_workbook():
+    """The committed EFO receipts workbook for the vintage in sources.yaml."""
+    return obr._fallback_workbook(load_config()["obr"]["efo_receipts"])
+
+
+def _sheet(wb, title: str):
+    (name,) = [n for n in wb.sheetnames if title in str(wb[n]["B2"].value)]
+    return wb[name]
+
+
+def _row(ws, label: str, columns: str) -> dict[int, float]:
+    row = obr._find_row(ws, label)
+    return {year: ws[f"{col}{row}"].value * 1e9 for col, year in zip(columns, YEARS)}
+
+
+def _stub_statistic(corporate: dict[int, float]):
+    return patch.object(
+        obr,
+        "corporate_sdlt_receipts",
+        lambda years=None: {year: corporate[year] for year in years},
+    )
+
+
+def test_target_is_obr_sdlt_less_corporate_statistic(receipts_workbook):
+    obr_sdlt = _row(
+        obr._find_receipts_sheet(receipts_workbook), "Stamp duty land tax", "DEFGHIJ"
+    )
+    target = {t.name: t for t in obr._parse_receipts(receipts_workbook)}["obr/sdlt"]
+    corporate = obr.corporate_sdlt_receipts(YEARS)
+
+    assert target.variable == "stamp_duty_land_tax"
+    assert sorted(target.values) == YEARS
+    for year in YEARS:
+        assert target.values[year] == pytest.approx(
+            obr_sdlt[year] - corporate[year], rel=1e-12
+        )
+
+
+def test_receipts_sdlt_line_is_the_property_type_total(receipts_workbook):
+    """The line the statistic is subtracted from is the total of the property
+    types policyengine-uk projects corporate SDLT with (EFO Table 3.7)."""
+    by_sector = _sheet(receipts_workbook, "Property transactions taxes")
+    assert _row(by_sector, "Total SDLT", "CDEFGHI") == pytest.approx(
+        _row(
+            obr._find_receipts_sheet(receipts_workbook),
+            "Stamp duty land tax",
+            "DEFGHIJ",
+        )
+    )
+
+
+def test_statistic_is_what_corporate_sdlt_adds():
+    """A household holding the whole corporate sector pays the statistic."""
+    from policyengine_uk import Simulation
+
+    simulation = Simulation(
+        situation={
+            "people": {"adult": {"age": dict.fromkeys(YEARS, 40)}},
+            "benunits": {"benunit": {"members": ["adult"]}},
+            "households": {
+                "household": {
+                    "members": ["adult"],
+                    "shareholding": dict.fromkeys(YEARS, 1.0),
+                }
+            },
+        }
+    )
+    statistic = obr.corporate_sdlt_receipts(YEARS)
+    for year in YEARS:
+        assert float(simulation.calculate("corporate_sdlt", year)[0]) == (
+            pytest.approx(statistic[year], rel=1e-6)
+        )
+
+
+_amounts = st.floats(min_value=0, max_value=1e12, allow_nan=False)
+
+
+@given(st.dictionaries(st.integers(2000, 2040), st.tuples(_amounts, _amounts)))
+def test_household_sdlt_is_obr_sdlt_less_statistic(pairs):
+    obr_sdlt = {year: max(pair) for year, pair in pairs.items()}
+    corporate = {year: min(pair) for year, pair in pairs.items()}
+    with _stub_statistic(corporate):
+        household = obr.household_sdlt(obr_sdlt)
+
+    assert household.keys() == obr_sdlt.keys()
+    for year, total in obr_sdlt.items():
+        assert 0 <= household[year] <= total
+        assert household[year] + corporate[year] == pytest.approx(total, abs=1e-3)
+
+
+@given(
+    st.dictionaries(st.integers(2000, 2040), _amounts, min_size=1),
+    st.floats(min_value=1, max_value=1e9),
+    st.data(),
+)
+def test_statistic_above_obr_sdlt_is_an_error(obr_sdlt, excess, data):
+    year = data.draw(st.sampled_from(sorted(obr_sdlt)))
+    corporate = {**dict.fromkeys(obr_sdlt, 0.0), year: obr_sdlt[year] + excess}
+    with _stub_statistic(corporate), pytest.raises(ValueError, match=str(year)):
+        obr.household_sdlt(obr_sdlt)
+
+
+def test_flat_statistic_warns(caplog):
+    flat = dict.fromkeys(YEARS, 3.13e9)
+    with _stub_statistic(flat), caplog.at_level(logging.WARNING):
+        obr.household_sdlt(dict.fromkeys(YEARS, 20e9))
+    assert "PolicyEngine/policyengine-uk#2322" in caplog.text
+
+
+def test_projected_statistic_does_not_warn(caplog):
+    projected = {year: 4e9 + 1e8 * (year - YEARS[0]) for year in YEARS}
+    with _stub_statistic(projected), caplog.at_level(logging.WARNING):
+        obr.household_sdlt(dict.fromkeys(YEARS, 20e9))
+    assert caplog.text == ""

policyengine_uk_data/tests/test_property_purchased.py

@@ -11,14 +11,19 @@
   https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/families/bulletins/familiesandhouseholds/2024
 - Rate: 1.1m / 28.6m = 3.85%
 
-Verification against official SDLT revenue (2024-25):
-- Official SDLT: £13.9bn (https://www.gov.uk/government/statistics/uk-stamp-tax-statistics)
-- With fix (3.85%): ~£15bn (close to official)
-- Without fix (100%): £370bn (26x too high)
+Households' SDLT is checked against the OBR's SDLT forecast less the SDLT
+policyengine-uk attributes to companies and other non-natural persons
+(`corporate_sdlt`), the figure calibration targets (#564). Before the 3.85%
+rate, every household was charged SDLT and the total was about £370bn.
 """
 
+import os
+
 import pytest
 
+from policyengine_uk_data.targets.sources import obr
+from policyengine_uk_data.targets.sources._common import load_config
+
 PROPERTY_PURCHASE_RATE = 0.0385
 
 
@@ -62,31 +67,39 @@ def test_property_purchased_has_some_true(baseline):
     assert true_count > 0, "No households have property_purchased=True"
 
 
-def test_sdlt_total_reasonable(baseline):
-    """Test that total SDLT revenue is realistic.
+# PR CI builds with TESTING=1 (32 calibration epochs instead of 512), so its
+# bound only catches order-of-magnitude errors. Full builds get the strict one.
+_SDLT_BOUNDS = (0.25, 4.0) if os.environ.get("TESTING") == "1" else (0.5, 1.5)
 
-    Official SDLT revenue (2024-25): £13.9bn
-    Source: https://www.gov.uk/government/statistics/uk-stamp-tax-statistics
 
-    Without fix (100% property_purchased=True): £370bn (26x too high)
-    With fix (3.85% rate): ~£15bn (close to official)
-    """
-    expected_sdlt = baseline.calculate("expected_sdlt", 2025).values
-    household_weight = baseline.calculate("household_weight", 2025).values
-    total_sdlt = (expected_sdlt * household_weight).sum()
+def _household_sdlt_target(year: int) -> float:
+    """The OBR SDLT calibration target: the OBR's SDLT less corporate SDLT."""
+    wb = obr._fallback_workbook(load_config()["obr"]["efo_receipts"])
+    return {t.name: t for t in obr._parse_receipts(wb)}["obr/sdlt"].values[year]
 
-    # Total SDLT should be within reasonable range of official figures
-    min_sdlt = 5e9  # £5bn minimum
-    max_sdlt = 50e9  # £50bn maximum (official is ~£14bn)
 
-    assert total_sdlt > min_sdlt, (
-        f"Total SDLT £{total_sdlt / 1e9:.1f}bn is too low "
-        f"(minimum expected: £{min_sdlt / 1e9:.1f}bn)"
+def test_household_sdlt_near_target(baseline):
+    """Households' SDLT is near the OBR's SDLT less corporate SDLT.
+
+    Compared with `stamp_duty_land_tax`, households' own SDLT: `expected_sdlt`
+    also included `corporate_sdlt` before PolicyEngine/policyengine-uk#2235.
+    """
+    weight = baseline.calculate("household_weight", 2025).values
+    household = (baseline.calculate("stamp_duty_land_tax", 2025).values * weight).sum()
+    target = _household_sdlt_target(2025)
+    low, high = _SDLT_BOUNDS
+
+    assert low < household / target < high, (
+        f"Households' SDLT is £{household / 1e9:.2f}bn against a target of "
+        f"£{target / 1e9:.2f}bn (the OBR's SDLT less corporate SDLT); the "
+        f"ratio must lie between {low} and {high}."
     )
 
-    assert total_sdlt < max_sdlt, (
-        f"Total SDLT £{total_sdlt / 1e9:.1f}bn is unrealistically high "
-        f"(maximum expected: £{max_sdlt / 1e9:.1f}bn). "
-        f"Official SDLT is ~£14bn. "
-        "This suggests property_purchased may be incorrectly set to True for all households."
+
+def test_corporate_sdlt_is_the_statistic_the_target_subtracts(baseline):
+    """With households on target, total SDLT is then the OBR's figure."""
+    weight = baseline.calculate("household_weight", 2025).values
+    corporate = (baseline.calculate("corporate_sdlt", 2025).values * weight).sum()
+    assert corporate == pytest.approx(
+        obr.corporate_sdlt_receipts([2025])[2025], rel=1e-6
     )