diff --git a/changelog.d/sdlt-target-excludes-corporate.fixed.md b/changelog.d/sdlt-target-excludes-corporate.fixed.md new file mode 100644 index 00000000..038c4efc --- /dev/null +++ b/changelog.d/sdlt-target-excludes-corporate.fixed.md @@ -0,0 +1 @@ +Calibrate households' SDLT (`stamp_duty_land_tax`) to the OBR's SDLT forecast less policyengine-uk's corporate SDLT statistic for the same year, instead of the whole OBR line. policyengine-uk adds SDLT paid by companies and other non-natural persons separately, as `corporate_sdlt`, so calibration counted that share twice: on the 1.58.0 dataset with PolicyEngine/policyengine-uk#2322, 2026-27 SDLT came to £23.29bn (£17.31bn households, £5.98bn corporate) against the OBR's £18.15bn. The 2026-27 household target becomes £12.17bn. Full builds now need a policyengine-uk with #2322, which projects the corporate statistic with the OBR forecast, and stop at the start with an earlier one, which holds it at £3.13bn in every year. diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py index 31008183..1c24e006 100644 --- a/policyengine_uk_data/datasets/create_datasets.py +++ b/policyengine_uk_data/datasets/create_datasets.py @@ -58,7 +58,8 @@ def _materialize_base_year_dataset(dataset, frs_release, uprate_dataset): def main(): """Create enhanced FRS dataset with rich progress tracking.""" try: - assert_local_build_environment() + # TESTING builds (PR CI) skip the full-build checks. + assert_local_build_environment(full_build=os.environ.get("TESTING", "0") != "1") from policyengine_uk.data import UKSingleYearDataset from policyengine_uk_data.datasets.disability_benefits import ( diff --git a/policyengine_uk_data/targets/sources/obr.py b/policyengine_uk_data/targets/sources/obr.py index 1b7d3661..5cc662ff 100644 --- a/policyengine_uk_data/targets/sources/obr.py +++ b/policyengine_uk_data/targets/sources/obr.py @@ -193,6 +193,65 @@ def _find_row(ws, label: str, col: str = "B", max_row: int = 80) -> int: raise ValueError(f"Row '{label}' not found in sheet") +def corporate_sdlt_receipts(years=None) -> dict[int, float]: + """policyengine-uk's SDLT receipts from non-natural persons, by year. + + These are the statistics `corporate_sdlt` allocates to households: each + household's `shareholding` divides its corporate-sector wealth by the + weighted total, so the weighted sum of `corporate_sdlt` is this statistic + whatever the weights. Read from the installed policyengine-uk, which + projects it from PolicyEngine/policyengine-uk#2322 on. Years are + PolicyEngine years (2025 is 2025-26); the default is the OBR's years. + """ + from policyengine_uk.system import system + + statistics = system.parameters.gov.hmrc.stamp_duty.statistics + return { + year: float( + statistics.residential.corporate.revenue(f"{year}-01-01") + + statistics.non_residential.corporate.revenue(f"{year}-01-01") + ) + for year in (_FY_COL_TO_YEAR.values() if years is None else years) + } + + +def corporate_sdlt_is_projected(corporate: dict[int, float]) -> bool: + """Whether the corporate SDLT statistic changes over the years given. + + policyengine-uk before PolicyEngine/policyengine-uk#2322 held it at its + 2020-21 value, £3.13bn, in every later year. + """ + return len(set(corporate.values())) > 1 + + +def household_sdlt(obr_sdlt: dict[int, float]) -> dict[int, float]: + """The OBR's SDLT receipts less policyengine-uk's corporate SDLT, by year. + + The OBR's SDLT line covers every buyer. policyengine-uk counts receipts + from non-natural persons (companies, partnerships, collective investment + schemes) separately, as `corporate_sdlt`, so targeting the household + variable `stamp_duty_land_tax` at the whole line counted them twice (#564). + """ + corporate = corporate_sdlt_receipts(obr_sdlt) + if len(corporate) > 1 and not corporate_sdlt_is_projected(corporate): + logger.warning( + "OBR receipts: policyengine-uk holds corporate SDLT at £%.2fbn in " + "every year from %d to %d, so it predates " + "PolicyEngine/policyengine-uk#2322 and the household SDLT target " + "overstates households' share. Full builds refuse it.", + next(iter(corporate.values())) / 1e9, + min(corporate), + max(corporate), + ) + household = {year: total - corporate[year] for year, total in obr_sdlt.items()} + negative = sorted(year for year, value in household.items() if value < 0) + if negative: + raise ValueError( + f"policyengine-uk's corporate SDLT exceeds the OBR's SDLT in {negative}" + ) + return household + + def _parse_receipts(wb: openpyxl.Workbook) -> list[Target]: """Parse tax receipts from the OBR EFO. @@ -269,6 +328,8 @@ def read_39(ws, row_num: int) -> dict[int, float]: try: row_num = _find_row(ws39, label, col="B", max_row=80) values = read_39(ws39, row_num) + if values and name == "sdlt": + values = household_sdlt(values) if values: recovered.append(label) targets.append( @@ -282,8 +343,8 @@ def read_39(ws, row_num: int) -> dict[int, float]: forecast_vintage=vintage, ) ) - except ValueError: - logger.warning("OBR receipts: row '%s' not found", label) + except ValueError as e: + logger.warning("OBR receipts: row '%s' not used: %s", label, e) # Guard on targets actually produced, not on exceptions caught: a layout # change that shifts the value columns finds every row and still yields diff --git a/policyengine_uk_data/tests/test_build_environment.py b/policyengine_uk_data/tests/test_build_environment.py index 3f867645..d6f5e100 100644 --- a/policyengine_uk_data/tests/test_build_environment.py +++ b/policyengine_uk_data/tests/test_build_environment.py @@ -29,3 +29,26 @@ def test_get_local_build_issues_accepts_supported_environment(): ) == [] ) + + +def test_get_local_build_issues_requires_projected_corporate_sdlt(): + """policyengine-uk before #2322 held corporate SDLT at £3.13bn.""" + issues = get_local_build_issues( + python_version=(3, 13, 7), + variable_names={"num_vehicles"}, + corporate_sdlt=dict.fromkeys(range(2024, 2031), 3.13e9), + ) + + assert len(issues) == 1 + assert "PolicyEngine/policyengine-uk#2322" in issues[0] + + +def test_get_local_build_issues_accepts_projected_corporate_sdlt(): + assert ( + get_local_build_issues( + python_version=(3, 13, 7), + variable_names={"num_vehicles"}, + corporate_sdlt={2024: 4.49e9, 2025: 4.81e9, 2026: 5.98e9}, + ) + == [] + ) diff --git a/policyengine_uk_data/tests/test_obr_receipts_sheet.py b/policyengine_uk_data/tests/test_obr_receipts_sheet.py index d8b2928d..54d047d1 100644 --- a/policyengine_uk_data/tests/test_obr_receipts_sheet.py +++ b/policyengine_uk_data/tests/test_obr_receipts_sheet.py @@ -102,8 +102,10 @@ def _receipts_wb(*, populate: bool) -> openpyxl.Workbook: row = 10 + offset ws[f"B{row}"] = label if populate: + # £100bn a year, above the corporate SDLT statistic the SDLT + # target subtracts. for col in ("D", "E", "F", "G", "H", "I", "J"): - ws[f"{col}{row}"] = 1.0 + ws[f"{col}{row}"] = 100.0 return wb diff --git a/policyengine_uk_data/tests/test_obr_sdlt_target.py b/policyengine_uk_data/tests/test_obr_sdlt_target.py new file mode 100644 index 00000000..6843e044 --- /dev/null +++ b/policyengine_uk_data/tests/test_obr_sdlt_target.py @@ -0,0 +1,142 @@ +"""The OBR SDLT target leaves out the corporate SDLT policyengine-uk adds (#564). + +The OBR's SDLT line covers every buyer. policyengine-uk adds SDLT paid by +non-natural persons as `corporate_sdlt` (its corporate statistic times each +household's shareholding), so household `stamp_duty_land_tax` is targeted at +the OBR line less that statistic. + +Invariants, for every year the OBR forecasts: +- target + corporate statistic = OBR SDLT; +- 0 <= target <= OBR SDLT whenever 0 <= statistic <= OBR SDLT, and a statistic + above the OBR figure is an error rather than a negative target; +- the statistic subtracted is the one `corporate_sdlt` adds. +""" + +import logging +from unittest.mock import patch + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from policyengine_uk_data.targets.sources import obr +from policyengine_uk_data.targets.sources._common import load_config + +YEARS = sorted(obr._FY_COL_TO_YEAR.values()) + + +@pytest.fixture(scope="module") +def receipts_workbook(): + """The committed EFO receipts workbook for the vintage in sources.yaml.""" + return obr._fallback_workbook(load_config()["obr"]["efo_receipts"]) + + +def _sheet(wb, title: str): + (name,) = [n for n in wb.sheetnames if title in str(wb[n]["B2"].value)] + return wb[name] + + +def _row(ws, label: str, columns: str) -> dict[int, float]: + row = obr._find_row(ws, label) + return {year: ws[f"{col}{row}"].value * 1e9 for col, year in zip(columns, YEARS)} + + +def _stub_statistic(corporate: dict[int, float]): + return patch.object( + obr, + "corporate_sdlt_receipts", + lambda years=None: {year: corporate[year] for year in years}, + ) + + +def test_target_is_obr_sdlt_less_corporate_statistic(receipts_workbook): + obr_sdlt = _row( + obr._find_receipts_sheet(receipts_workbook), "Stamp duty land tax", "DEFGHIJ" + ) + target = {t.name: t for t in obr._parse_receipts(receipts_workbook)}["obr/sdlt"] + corporate = obr.corporate_sdlt_receipts(YEARS) + + assert target.variable == "stamp_duty_land_tax" + assert sorted(target.values) == YEARS + for year in YEARS: + assert target.values[year] == pytest.approx( + obr_sdlt[year] - corporate[year], rel=1e-12 + ) + + +def test_receipts_sdlt_line_is_the_property_type_total(receipts_workbook): + """The line the statistic is subtracted from is the total of the property + types policyengine-uk projects corporate SDLT with (EFO Table 3.7).""" + by_sector = _sheet(receipts_workbook, "Property transactions taxes") + assert _row(by_sector, "Total SDLT", "CDEFGHI") == pytest.approx( + _row( + obr._find_receipts_sheet(receipts_workbook), + "Stamp duty land tax", + "DEFGHIJ", + ) + ) + + +def test_statistic_is_what_corporate_sdlt_adds(): + """A household holding the whole corporate sector pays the statistic.""" + from policyengine_uk import Simulation + + simulation = Simulation( + situation={ + "people": {"adult": {"age": dict.fromkeys(YEARS, 40)}}, + "benunits": {"benunit": {"members": ["adult"]}}, + "households": { + "household": { + "members": ["adult"], + "shareholding": dict.fromkeys(YEARS, 1.0), + } + }, + } + ) + statistic = obr.corporate_sdlt_receipts(YEARS) + for year in YEARS: + assert float(simulation.calculate("corporate_sdlt", year)[0]) == ( + pytest.approx(statistic[year], rel=1e-6) + ) + + +_amounts = st.floats(min_value=0, max_value=1e12, allow_nan=False) + + +@given(st.dictionaries(st.integers(2000, 2040), st.tuples(_amounts, _amounts))) +def test_household_sdlt_is_obr_sdlt_less_statistic(pairs): + obr_sdlt = {year: max(pair) for year, pair in pairs.items()} + corporate = {year: min(pair) for year, pair in pairs.items()} + with _stub_statistic(corporate): + household = obr.household_sdlt(obr_sdlt) + + assert household.keys() == obr_sdlt.keys() + for year, total in obr_sdlt.items(): + assert 0 <= household[year] <= total + assert household[year] + corporate[year] == pytest.approx(total, abs=1e-3) + + +@given( + st.dictionaries(st.integers(2000, 2040), _amounts, min_size=1), + st.floats(min_value=1, max_value=1e9), + st.data(), +) +def test_statistic_above_obr_sdlt_is_an_error(obr_sdlt, excess, data): + year = data.draw(st.sampled_from(sorted(obr_sdlt))) + corporate = {**dict.fromkeys(obr_sdlt, 0.0), year: obr_sdlt[year] + excess} + with _stub_statistic(corporate), pytest.raises(ValueError, match=str(year)): + obr.household_sdlt(obr_sdlt) + + +def test_flat_statistic_warns(caplog): + flat = dict.fromkeys(YEARS, 3.13e9) + with _stub_statistic(flat), caplog.at_level(logging.WARNING): + obr.household_sdlt(dict.fromkeys(YEARS, 20e9)) + assert "PolicyEngine/policyengine-uk#2322" in caplog.text + + +def test_projected_statistic_does_not_warn(caplog): + projected = {year: 4e9 + 1e8 * (year - YEARS[0]) for year in YEARS} + with _stub_statistic(projected), caplog.at_level(logging.WARNING): + obr.household_sdlt(dict.fromkeys(YEARS, 20e9)) + assert caplog.text == "" diff --git a/policyengine_uk_data/tests/test_property_purchased.py b/policyengine_uk_data/tests/test_property_purchased.py index 0c742180..3a1fe59a 100644 --- a/policyengine_uk_data/tests/test_property_purchased.py +++ b/policyengine_uk_data/tests/test_property_purchased.py @@ -11,14 +11,19 @@ https://www.ons.gov.uk/peoplepopulationandcommunity/birthsdeathsandmarriages/families/bulletins/familiesandhouseholds/2024 - Rate: 1.1m / 28.6m = 3.85% -Verification against official SDLT revenue (2024-25): -- Official SDLT: £13.9bn (https://www.gov.uk/government/statistics/uk-stamp-tax-statistics) -- With fix (3.85%): ~£15bn (close to official) -- Without fix (100%): £370bn (26x too high) +Households' SDLT is checked against the OBR's SDLT forecast less the SDLT +policyengine-uk attributes to companies and other non-natural persons +(`corporate_sdlt`), the figure calibration targets (#564). Before the 3.85% +rate, every household was charged SDLT and the total was about £370bn. """ +import os + import pytest +from policyengine_uk_data.targets.sources import obr +from policyengine_uk_data.targets.sources._common import load_config + PROPERTY_PURCHASE_RATE = 0.0385 @@ -62,31 +67,39 @@ def test_property_purchased_has_some_true(baseline): assert true_count > 0, "No households have property_purchased=True" -def test_sdlt_total_reasonable(baseline): - """Test that total SDLT revenue is realistic. +# PR CI builds with TESTING=1 (32 calibration epochs instead of 512), so its +# bound only catches order-of-magnitude errors. Full builds get the strict one. +_SDLT_BOUNDS = (0.25, 4.0) if os.environ.get("TESTING") == "1" else (0.5, 1.5) - Official SDLT revenue (2024-25): £13.9bn - Source: https://www.gov.uk/government/statistics/uk-stamp-tax-statistics - Without fix (100% property_purchased=True): £370bn (26x too high) - With fix (3.85% rate): ~£15bn (close to official) - """ - expected_sdlt = baseline.calculate("expected_sdlt", 2025).values - household_weight = baseline.calculate("household_weight", 2025).values - total_sdlt = (expected_sdlt * household_weight).sum() +def _household_sdlt_target(year: int) -> float: + """The OBR SDLT calibration target: the OBR's SDLT less corporate SDLT.""" + wb = obr._fallback_workbook(load_config()["obr"]["efo_receipts"]) + return {t.name: t for t in obr._parse_receipts(wb)}["obr/sdlt"].values[year] - # Total SDLT should be within reasonable range of official figures - min_sdlt = 5e9 # £5bn minimum - max_sdlt = 50e9 # £50bn maximum (official is ~£14bn) - assert total_sdlt > min_sdlt, ( - f"Total SDLT £{total_sdlt / 1e9:.1f}bn is too low " - f"(minimum expected: £{min_sdlt / 1e9:.1f}bn)" +def test_household_sdlt_near_target(baseline): + """Households' SDLT is near the OBR's SDLT less corporate SDLT. + + Compared with `stamp_duty_land_tax`, households' own SDLT: `expected_sdlt` + also included `corporate_sdlt` before PolicyEngine/policyengine-uk#2235. + """ + weight = baseline.calculate("household_weight", 2025).values + household = (baseline.calculate("stamp_duty_land_tax", 2025).values * weight).sum() + target = _household_sdlt_target(2025) + low, high = _SDLT_BOUNDS + + assert low < household / target < high, ( + f"Households' SDLT is £{household / 1e9:.2f}bn against a target of " + f"£{target / 1e9:.2f}bn (the OBR's SDLT less corporate SDLT); the " + f"ratio must lie between {low} and {high}." ) - assert total_sdlt < max_sdlt, ( - f"Total SDLT £{total_sdlt / 1e9:.1f}bn is unrealistically high " - f"(maximum expected: £{max_sdlt / 1e9:.1f}bn). " - f"Official SDLT is ~£14bn. " - "This suggests property_purchased may be incorrectly set to True for all households." + +def test_corporate_sdlt_is_the_statistic_the_target_subtracts(baseline): + """With households on target, total SDLT is then the OBR's figure.""" + weight = baseline.calculate("household_weight", 2025).values + corporate = (baseline.calculate("corporate_sdlt", 2025).values * weight).sum() + assert corporate == pytest.approx( + obr.corporate_sdlt_receipts([2025])[2025], rel=1e-6 ) diff --git a/policyengine_uk_data/utils/build_environment.py b/policyengine_uk_data/utils/build_environment.py index aa4f0a5e..b6e128a5 100644 --- a/policyengine_uk_data/utils/build_environment.py +++ b/policyengine_uk_data/utils/build_environment.py @@ -5,7 +5,14 @@ def get_local_build_issues( python_version: tuple[int, ...] | None = None, variable_names: Iterable[str] | None = None, + corporate_sdlt: dict[int, float] | None = None, ) -> list[str]: + """Problems with the build environment. + + ``corporate_sdlt`` is the resolved policyengine-uk's corporate SDLT + statistic by year (``obr.corporate_sdlt_receipts``). It is checked only + when given, which full builds do. + """ issues: list[str] = [] if python_version is None: @@ -30,11 +37,32 @@ def get_local_build_issues( "PYTHONPATH." ) + if corporate_sdlt is not None: + from policyengine_uk_data.targets.sources.obr import ( + corporate_sdlt_is_projected, + ) + + if not corporate_sdlt_is_projected(corporate_sdlt): + issues.append( + "The resolved policyengine-uk holds corporate SDLT at " + f"£{next(iter(corporate_sdlt.values())) / 1e9:.2f}bn in every " + f"year from {min(corporate_sdlt)} to {max(corporate_sdlt)}. " + "Full builds need a version with " + "PolicyEngine/policyengine-uk#2322, which projects it with the " + "OBR forecast: households' SDLT is calibrated to the OBR total " + "less this statistic, so a stale one overstates it." + ) + return issues -def assert_local_build_environment() -> None: - issues = get_local_build_issues() +def assert_local_build_environment(full_build: bool = False) -> None: + corporate_sdlt = None + if full_build: + from policyengine_uk_data.targets.sources.obr import corporate_sdlt_receipts + + corporate_sdlt = corporate_sdlt_receipts() + issues = get_local_build_issues(corporate_sdlt=corporate_sdlt) if not issues: return