diff --git a/changelog.d/556.added.md b/changelog.d/556.added.md new file mode 100644 index 00000000..d33da552 --- /dev/null +++ b/changelog.d/556.added.md @@ -0,0 +1 @@ +Impute seeded additional-dwelling and non-residential purchase prices from household property stocks, fitting explicit HMRC household transaction and value assumptions with a hard stock cap. diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py index 31008183..32ab5768 100644 --- a/policyengine_uk_data/datasets/create_datasets.py +++ b/policyengine_uk_data/datasets/create_datasets.py @@ -111,6 +111,7 @@ def main(): "Impute student loan plan", "Assign Pension Credit take-up", "Clone and assign OA geography", + "Impute property purchases", "Calibrate constituency weights", "Calibrate local authority weights", "Export long local geography weights", @@ -163,6 +164,7 @@ def main(): impute_salary_sacrifice, impute_student_loan_plan, uprate_property_by_region, + impute_property_purchases, ) # Apply imputations with progress tracking @@ -247,6 +249,12 @@ def main(): ) update_dataset(materialize_calibration_step, "completed") + # Stocks and final household geography now exist at the calibration + # year. Supply independent purchase inputs before fitting SDLT. + update_dataset("Impute property purchases", "processing") + frs = impute_property_purchases(frs) + update_dataset("Impute property purchases", "completed") + # Calibrate constituency weights with nested progress update_dataset("Calibrate constituency weights", "processing") @@ -417,7 +425,7 @@ def main(): "tiny_base_dataset": frs_release.tiny_base_dataset_file, "tiny_enhanced_dataset": frs_release.tiny_enhanced_dataset_file, "long_geography_weights": "local_geography_weights.csv.gz", - "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan", + "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan, property_purchases", "calibration": "national, LA and constituency targets", "pension_credit_take_up": pension_credit_takeup, }, diff --git a/policyengine_uk_data/datasets/imputations/__init__.py b/policyengine_uk_data/datasets/imputations/__init__.py index 736941f1..ba75f56b 100644 --- a/policyengine_uk_data/datasets/imputations/__init__.py +++ b/policyengine_uk_data/datasets/imputations/__init__.py @@ -8,3 +8,4 @@ from .salary_sacrifice import impute_salary_sacrifice from .student_loans import impute_student_loan_plan from .regional_property_uprating import uprate_property_by_region +from .property_purchases import impute_property_purchases diff --git a/policyengine_uk_data/datasets/imputations/property_purchases.py b/policyengine_uk_data/datasets/imputations/property_purchases.py new file mode 100644 index 00000000..9f732722 --- /dev/null +++ b/policyengine_uk_data/datasets/imputations/property_purchases.py @@ -0,0 +1,147 @@ +"""Impute one annual purchase from each kind of household property stock. + +HMRC targets cover England and Northern Ireland. Independent seeded event +draws are calibrated there; the resulting rates also apply in Scotland and +Wales. Prices are separate seeded draws, never the household's entire stock +by default. Assumptions and sources live in stochastic parameters. +""" + +import numpy as np + +from policyengine_uk_data.parameters import load_parameter + +SDLT_REGIONS = ( + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", + "NORTHERN_IRELAND", +) + + +def impute_property_purchase_values( + stock: np.ndarray, + weights: np.ndarray, + sdlt_mask: np.ndarray, + year: int, + property_type: str, +) -> np.ndarray: + """Return seeded purchase prices, fitting weighted events and capped value. + + Purchase hazard is proportional to stock, an explicit portfolio-turnover + assumption. The closest weighted count determines the event threshold. + The value target is fitted by multiplying sampled band-average prices, + with the stock cap applied throughout. Zero-weight donors never purchase. + """ + stock = np.asarray(stock, dtype=float) + weights = np.asarray(weights, dtype=float) + sdlt_mask = np.asarray(sdlt_mask, dtype=bool) + if ( + stock.ndim != 1 + or stock.shape != weights.shape + or stock.shape != sdlt_mask.shape + ): + raise ValueError( + "Property stocks, weights and SDLT mask must be aligned vectors" + ) + if ( + not np.all(np.isfinite(stock)) + or not np.all(np.isfinite(weights)) + or np.any(stock < 0) + or np.any(weights < 0) + ): + raise ValueError("Property stocks and weights must be finite and non-negative") + + assumptions = load_parameter("stochastic", "additional_property_purchases", year)[ + property_type + ] + target_count = assumptions["transactions"] + target_value = assumptions["value"] + minimum_price = assumptions["minimum_price"] + owners = (stock >= minimum_price) & (weights > 0) + eligible = np.flatnonzero(owners & sdlt_mask) + if weights[eligible].sum() < target_count: + raise ValueError("Insufficient weighted property owners for purchase target") + + reference_stock = np.average(stock[eligible], weights=weights[eligible]) + hazard = (np.maximum(stock, minimum_price) / reference_stock) ** assumptions[ + "purchase_stock_exponent" + ] + uniform = np.random.default_rng(assumptions["purchase_seed"]).random(len(stock)) + events = -np.log1p(-uniform) / hazard + ranked = eligible[np.argsort(events[eligible], kind="stable")] + counts = np.concatenate(([0.0], np.cumsum(weights[ranked]))) + n_purchases = int(np.argmin(np.abs(counts - target_count))) + if n_purchases == len(ranked): + threshold = np.inf + elif n_purchases == 0: + threshold = 0.0 + else: + threshold = (events[ranked[n_purchases - 1]] + events[ranked[n_purchases]]) / 2 + purchased = owners & (events < threshold) + fitted_weights = weights * (purchased & sdlt_mask) + capacity = np.dot(stock, fitted_weights) + if capacity < target_value: + raise ValueError( + "Selected property stock cannot support the purchase value target" + ) + if minimum_price * fitted_weights.sum() > target_value: + raise ValueError("Purchase value target is below the minimum transaction price") + + bands = assumptions["price_bands"] + band_counts = np.array([band["transactions"] for band in bands], dtype=float) + band_prices = np.array([band["value"] for band in bands]) / band_counts + price_rng = np.random.default_rng(assumptions["price_seed"]) + prices = price_rng.choice( + band_prices, size=len(stock), p=band_counts / band_counts.sum() + ) + + def capped_prices(scale): + return np.minimum(stock, np.maximum(minimum_price, prices * scale)) + + def weighted_value(scale): + return np.dot(capped_prices(scale), fitted_weights) + + lower, upper = 0.0, 1.0 + while weighted_value(upper) < target_value: + upper *= 2 + for _ in range(80): + midpoint = (lower + upper) / 2 + if weighted_value(midpoint) < target_value: + lower = midpoint + else: + upper = midpoint + return np.where(purchased, capped_prices(upper), 0.0) + + +def impute_property_purchases(dataset, year=None): + """Add annual purchase inputs after wealth imputation, before calibration.""" + dataset = dataset.copy() + year = int(dataset.time_period) if year is None else year + household = dataset.household + sdlt_mask = household["region"].astype(str).isin(SDLT_REGIONS).values + for property_type, stock_name, purchase_name in ( + ( + "additional_residential", + "other_residential_property_value", + "additional_residential_property_purchased", + ), + ( + "non_residential", + "non_residential_property_value", + "non_residential_property_purchased", + ), + ): + household[purchase_name] = impute_property_purchase_values( + household[stock_name].values, + household["household_weight"].values, + sdlt_mask, + year, + property_type, + ) + return dataset diff --git a/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml new file mode 100644 index 00000000..c6671b70 --- /dev/null +++ b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml @@ -0,0 +1,104 @@ +description: Household additional-dwelling and non-residential purchase assumptions +metadata: + label: Additional property purchases + reference: + - title: HMRC Annual Stamp Tax Statistics 2024 to 2025, Tables 6b, 8a and 8b + href: https://www.gov.uk/government/statistics/uk-stamp-tax-statistics + - title: HMRC Annual Stamp Tax Statistics 2024 to 2025, corrected tables + href: https://assets.publishing.service.gov.uk/media/69aef72fc78869bf8eb8a589/Annual_Stamps_2024_25_correction.ods + +# HMRC SDLT tables cover England and Northern Ireland (E+NI), transactions +# of at least GBP 40,000, and financial years ending in the following March. +# Transaction counts are rounded to 100; values and receipts to GBP 5 million. +# All values below are pounds. Imputation fits E+NI weighted counts and values. +# Mean annual purchase rate is target transactions / eligible E+NI weighted +# owners of stock of at least GBP 40,000. Stock-proportional purchase hazard +# (exponent 1) assumes larger portfolios are more likely to trade each year; +# it is a modelling choice, not an observed household transaction propensity. +# Independent seeded exponential event scores divided by relative stock +# select the closest attainable weighted count. The same hazard and threshold +# apply elsewhere in the UK, an extrapolation without devolved targets. +# Uniform owner selection on the local enhanced FRS 2023-24 selected too little +# stock to support the assumed HRAD value. Stock weighting addresses that +# feasibility issue without tuning seeds to a particular dataset. +# Prices start from a seeded draw among band means (band value / transactions). +# A common scale then fits the weighted purchase value with price equal to +# min(stock, max(40,000, scale * draw)), so prices stay within HMRC's +# transaction coverage and never exceed the owner's stock. +# This is a capped, calibrated band-mean method, not an exact draw from the +# price-band distribution conditional on stock. Rounded band counts are +# normalised by their sum rather than by the headline count. +# +# Household HRAD is not separately published. The modelling choice is neutral +# allocation using individuals' shares of all residential buyers in Table 8a: +# transactions = 211,700 * 925,100 / (925,100 + 124,500) = 186,588.8624; +# value = 77,970,000,000 * 335,515 / (335,515 + 44,690) = 68,805,261,766.6785. +# Revenue is a reference benchmark, not fitted by imputation: +# 5,435,000,000 * 9,145 / (9,145 + 1,230) = 4,790,657,831.3253. +# This allocation is an assumption, not an observed household HRAD series. +# Table 6b reports all-buyer gross HRAD SDLT before replacement-home refunds; +# its surcharge-only receipts are GBP 2.790bn. The independent household lower +# bounds implied by Tables 6b/8a are 87,200 transactions, GBP 33.280bn value, +# and GBP 4.205bn gross receipts. They are bounds, not the selected targets. +# For review, 2024-25 household HRAD targets and price bands are held flat across +# survey/calibration years: there is no annual official household HRAD series +# and this assumption is not a forecast. Non-residential targets use the +# directly observed Table 8b individuals rows for 2023-24 and 2024-25, replacing +# the older household rows in policyengine-uk's stamp_duty/statistics.yaml. +# Their price bands also use individuals only. The 2024-25 band counts sum to +# 25,100 rather than the 25,000 headline because of rounding; probabilities +# use that band sum. Stock caps and the fitted scale modify the band distribution. +values: + 2013-01-01: + additional_residential: &additional_residential + transactions: 186588.8624237805 + value: 68805261766.6785 + revenue: 4790657831.325301 + minimum_price: 40000 + purchase_stock_exponent: 1 + purchase_seed: 55601 + price_seed: 55602 + # Table 6b 2024-25: last band combines all five published bands over GBP 2m. + # Band sums differ slightly from headline totals because of rounding. + price_bands: + - {transactions: 113500, value: 16635000000} # Up to GBP 250,000 + - {transactions: 60700, value: 21510000000} # GBP 250,001-500,000 + - {transactions: 27500, value: 18770000000} # GBP 500,001-1,000,000 + - {transactions: 5300, value: 6545000000} # GBP 1,000,001-1,500,000 + - {transactions: 2000, value: 3515000000} # GBP 1,500,001-2,000,000 + - {transactions: 2600, value: 10990000000} # Over GBP 2,000,000 + non_residential: + # 2023-24 Table 8b individuals: 24,800 / GBP 6.955bn / GBP 230m. + # Table 8b individuals' price-band means, before stock-capped scaling. + transactions: 24800 + value: 6955000000 + revenue: 230000000 + minimum_price: 40000 + purchase_stock_exponent: 1 + purchase_seed: 55603 + price_seed: 55604 + price_bands: + - {transactions: 18800, value: 1545000000} # Up to GBP 250,000 + - {transactions: 3300, value: 1150000000} # GBP 250,001-500,000 + - {transactions: 1500, value: 1085000000} # GBP 500,001-1,000,000 + - {transactions: 500, value: 640000000} # GBP 1,000,001-1,500,000 + - {transactions: 200, value: 415000000} # GBP 1,500,001-2,000,000 + - {transactions: 500, value: 2120000000} # Over GBP 2,000,000 + 2024-01-01: + additional_residential: *additional_residential + non_residential: + # 2024-25 Table 8b individuals: 25,000 / GBP 7.640bn / GBP 245m. + transactions: 25000 + value: 7640000000 + revenue: 245000000 + minimum_price: 40000 + purchase_stock_exponent: 1 + purchase_seed: 55603 + price_seed: 55604 + price_bands: + - {transactions: 18800, value: 1560000000} # Up to GBP 250,000 + - {transactions: 3400, value: 1210000000} # GBP 250,001-500,000 + - {transactions: 1600, value: 1100000000} # GBP 500,001-1,000,000 + - {transactions: 500, value: 650000000} # GBP 1,000,001-1,500,000 + - {transactions: 300, value: 445000000} # GBP 1,500,001-2,000,000 + - {transactions: 500, value: 2675000000} # Over GBP 2,000,000 diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 149ef19f..5e98e38b 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -1,4 +1,5 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 +additional_residential_property_purchased,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 afcs_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 alcohol_and_tobacco_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 benunit_rent,1.0,1.0,1.0,1.11,1.184,1.223,1.275,1.312,1.351,1.392,1.392,1.392,1.392,1.392,1.392 @@ -49,6 +50,7 @@ miscellaneous_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384 mortgage_capital_repayment,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 mortgage_interest_repayment,1.0,1.0,1.262,1.874,2.288,2.599,2.927,3.167,3.3,3.455,3.455,3.455,3.455,3.455,3.455 net_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +non_residential_property_purchased,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 non_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 956f740d..9384e937 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -1,4 +1,5 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 +additional_residential_property_purchased,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 afcs_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 alcohol_and_tobacco_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 benunit_rent,0,0.0,0.0,0.11,0.067,0.033,0.043,0.029,0.03,0.03,0.0,0.0,0.0,0.0,0.0 @@ -49,6 +50,7 @@ miscellaneous_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0. mortgage_capital_repayment,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 mortgage_interest_repayment,0,0.0,0.262,0.485,0.221,0.136,0.126,0.082,0.042,0.047,0.0,0.0,0.0,0.0,0.0 net_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +non_residential_property_purchased,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 non_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_additional_property_purchased.py b/policyengine_uk_data/tests/test_additional_property_purchased.py new file mode 100644 index 00000000..c0894c95 --- /dev/null +++ b/policyengine_uk_data/tests/test_additional_property_purchased.py @@ -0,0 +1,298 @@ +"""Additional-property purchases are reproducible, stock capped and calibrated. + +Synthetic counts tolerate 10% error and value fits have relative tolerance +1e-8. Saved enhanced FRS aggregates tolerate 20% after weight calibration. +Dataset assertions report only aggregate diagnostics, never household rows. +""" + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.datasets.imputations.property_purchases import ( + impute_property_purchase_values, + impute_property_purchases, +) +from policyengine_uk_data.parameters import load_parameter + +PROPERTY_TYPES = ( + ( + "additional_residential", + "other_residential_property_value", + "additional_residential_property_purchased", + ), + ( + "non_residential", + "non_residential_property_value", + "non_residential_property_purchased", + ), +) +PROPERTY_TYPE_NAMES = tuple(item[0] for item in PROPERTY_TYPES) +RELEASE_YEARS = sorted( + {CURRENT_FRS_RELEASE.base_year, CURRENT_FRS_RELEASE.calibration_year} +) +SYNTHETIC_YEARS = sorted({2023, *RELEASE_YEARS}) + + +@pytest.fixture +def synthetic_households(): + """Heterogeneous owners, weights and nations with ample purchase capacity.""" + stock = np.geomspace(50_000.0, 50_000_000.0, 4_000) + stock[::5] = 0.0 + stock[1::41] = 10_000.0 + weights = np.linspace(100.0, 500.0, len(stock)) + weights[::11] = 0.0 + sdlt_mask = np.arange(len(stock)) % 7 != 0 + return stock, weights, sdlt_mask + + +def _targets(property_type, year): + return load_parameter("stochastic", "additional_property_purchases", year)[ + property_type + ] + + +def _assert_aggregate_targets( + prices, weights, sdlt_mask, targets, tolerance, value_tolerance=None +): + value_tolerance = tolerance if value_tolerance is None else value_tolerance + transactions = float(np.sum(weights[(prices > 0) & sdlt_mask])) + value = float(np.dot(prices[sdlt_mask], weights[sdlt_mask])) + assert transactions == pytest.approx(targets["transactions"], rel=tolerance), ( + f"Weighted transactions {transactions:,.0f}; " + f"target {targets['transactions']:,.0f}; tolerance {tolerance:.0%}" + ) + assert value == pytest.approx(targets["value"], rel=value_tolerance), ( + f"Weighted purchase value £{value / 1e9:.3f}bn; " + f"target £{targets['value'] / 1e9:.3f}bn; " + f"relative tolerance {value_tolerance:g}" + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +def test_purchase_imputation_is_independent_of_global_rng( + synthetic_households, property_type +): + stock, weights, sdlt_mask = synthetic_households + state = np.random.get_state() + try: + np.random.seed(1) + first = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, property_type + ) + np.random.seed(999) + np.random.random(10_000) + second = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, property_type + ) + finally: + np.random.set_state(state) + np.testing.assert_array_equal(first, second) + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +@pytest.mark.parametrize("year", SYNTHETIC_YEARS) +def test_purchase_prices_and_weighted_targets( + synthetic_households, property_type, year +): + stock, weights, sdlt_mask = synthetic_households + prices = impute_property_purchase_values( + stock, weights, sdlt_mask, year, property_type + ) + assert np.all(np.isfinite(prices)) + assert np.all(prices >= 0) + assert np.all(prices <= stock) + assert np.all(prices[(stock < 40_000) | (weights == 0)] == 0) + assert np.all(prices[prices > 0] >= 40_000) + assert np.any(prices > 0) + assert np.any((prices > 0) & (prices < stock)) + _assert_aggregate_targets( + prices, + weights, + sdlt_mask, + _targets(property_type, year), + tolerance=0.10, + value_tolerance=1e-8, + ) + + +@pytest.mark.unit +def test_property_types_have_independent_purchase_draws( + synthetic_households, monkeypatch +): + from policyengine_uk_data.datasets.imputations import property_purchases + + original = load_parameter("stochastic", "additional_property_purchases", 2024) + # Equal targets and price bands isolate the independence of event seeds. + assumptions = { + property_type: { + **original["additional_residential"], + "purchase_seed": original[property_type]["purchase_seed"], + "price_seed": original[property_type]["price_seed"], + } + for property_type, _, _ in PROPERTY_TYPES + } + monkeypatch.setattr(property_purchases, "load_parameter", lambda *args: assumptions) + stock, weights, sdlt_mask = synthetic_households + residential = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, "additional_residential" + ) + non_residential = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, "non_residential" + ) + assert np.any((residential > 0) != (non_residential > 0)) + + +@dataclass +class _HouseholdDataset: + household: pd.DataFrame + time_period: int = 2024 + + def copy(self): + return _HouseholdDataset(self.household.copy(), self.time_period) + + +@pytest.mark.unit +def test_purchase_inputs_do_not_depend_on_main_residence_purchase( + synthetic_households, +): + stock, weights, sdlt_mask = synthetic_households + original = _HouseholdDataset( + pd.DataFrame( + { + "region": np.where(sdlt_mask, "LONDON", "SCOTLAND"), + "household_weight": weights, + "other_residential_property_value": stock, + "non_residential_property_value": stock, + "property_purchased": False, + } + ) + ) + changed = original.copy() + changed.household["property_purchased"] = True + first = impute_property_purchases(original) + second = impute_property_purchases(changed) + for property_type, _, purchase_name in PROPERTY_TYPES: + assert purchase_name not in original.household + np.testing.assert_array_equal( + first.household[purchase_name], second.household[purchase_name] + ) + _assert_aggregate_targets( + first.household[purchase_name].to_numpy(), + weights, + sdlt_mask, + _targets(property_type, 2024), + tolerance=0.10, + value_tolerance=1e-8, + ) + assert not original.household.property_purchased.any() + assert second.household.property_purchased.all() + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +def test_insufficient_owner_population_fails(property_type): + with pytest.raises(ValueError, match="Insufficient weighted property owners"): + impute_property_purchase_values( + np.full(100, 1_000_000.0), + np.ones(100), + np.ones(100, dtype=bool), + 2024, + property_type, + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +def test_stock_cap_prevents_infeasible_value_target( + synthetic_households, property_type +): + stock, weights, sdlt_mask = synthetic_households + with pytest.raises(ValueError, match="stock cannot support"): + impute_property_purchase_values( + np.where(stock > 0, 40_000.0, 0.0), + weights, + sdlt_mask, + 2024, + property_type, + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "stock,weights", + [ + (np.array([np.nan]), np.ones(1)), + (np.ones(1), np.array([np.inf])), + (np.array([-1.0]), np.ones(1)), + (np.ones(1), np.array([-1.0])), + ], +) +def test_invalid_stock_or_weights_fail(stock, weights): + with pytest.raises(ValueError, match="finite and non-negative"): + impute_property_purchase_values( + stock, weights, np.ones(1, dtype=bool), 2024, "additional_residential" + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("property_type,stock_name,purchase_name", PROPERTY_TYPES) +@pytest.mark.parametrize("year", RELEASE_YEARS) +def test_saved_enhanced_frs_purchase_inputs( + enhanced_frs, baseline, property_type, stock_name, purchase_name, year +): + """A rebuilt dataset retains the input and targets after calibration.""" + input_columns = set(enhanced_frs.household.columns) + assert purchase_name in input_columns, ( + f"Enhanced FRS lacks {purchase_name}; rebuild with purchase imputation" + ) + prices = baseline.calculate(purchase_name, year).values + stock = baseline.calculate(stock_name, year).values + weights = baseline.calculate("household_weight", year).values + country = baseline.calculate("country", year, map_to="household").values + sdlt_mask = np.isin(country, ["ENGLAND", "NORTHERN_IRELAND"]) + # Store only aggregate booleans in assertions: pytest otherwise includes + # the household arrays in its rewritten assertion failure diagnostics. + finite = bool(np.all(np.isfinite(prices))) + non_negative = bool(np.all(prices >= 0)) + stock_capped = bool(np.all(prices <= stock + 0.01)) + owners_only = bool(np.all(prices[stock <= 0] == 0)) + assert finite, "Non-finite purchase inputs" + assert non_negative, "Negative purchase prices" + assert stock_capped, "Purchase prices exceed property stocks" + assert owners_only, "Non-owners purchase property" + _assert_aggregate_targets( + prices, weights, sdlt_mask, _targets(property_type, year), tolerance=0.20 + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("year", RELEASE_YEARS) +def test_saved_enhanced_frs_higher_rates_sdlt(baseline, year): + """Gross HRAD SDLT is non-zero and near the allocated household benchmark. + + The 30% tolerance allows price-distribution and calibration differences, and + the annual model scale versus HMRC's fiscal-year rate changes and refunds. + """ + prices = baseline.calculate( + "additional_residential_property_purchased", year + ).values + weights = baseline.calculate("household_weight", year).values + country = baseline.calculate("country", year, map_to="household").values + sdlt_mask = np.isin(country, ["ENGLAND", "NORTHERN_IRELAND"]) + additional = baseline.tax_benefit_system.parameters( + f"{year}-01-01" + ).gov.hmrc.stamp_duty.residential.purchase.additional + liability = np.where(prices >= additional.min, additional.rate.calc(prices), 0.0) + revenue = float(np.dot(liability[sdlt_mask], weights[sdlt_mask])) + target = _targets("additional_residential", year)["revenue"] + assert revenue > 0, "Additional-dwelling purchases have no higher-rates SDLT" + assert revenue == pytest.approx(target, rel=0.30), ( + f"Weighted higher-rates SDLT £{revenue / 1e9:.3f}bn; " + f"household benchmark £{target / 1e9:.3f}bn; tolerance 30%" + ) diff --git a/policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py b/policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py new file mode 100644 index 00000000..972754e1 --- /dev/null +++ b/policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py @@ -0,0 +1,72 @@ +"""Purchase inputs must retain their stock caps during year materialisation.""" + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.utils.uprating import ( + _apply_property_purchase_uprating_override, + uprate_dataset, +) + +PROPERTY_INPUTS = ( + ("additional_residential_property_purchased", "other_residential_property_value"), + ("non_residential_property_purchased", "non_residential_property_value"), +) + + +@pytest.mark.unit +@pytest.mark.parametrize("purchase_name,stock_name", PROPERTY_INPUTS) +@pytest.mark.parametrize("purchase_exists", [False, True]) +def test_uprating_generator_adds_or_replaces_purchase_index( + purchase_name, stock_name, purchase_exists +): + factors = pd.DataFrame([[1.2, 1.3]], index=[stock_name], columns=[2024, 2025]) + if purchase_exists: + factors.loc[purchase_name] = [9.0, 10.0] + updated = _apply_property_purchase_uprating_override(factors) + np.testing.assert_array_equal(updated.loc[purchase_name], [1.2, 1.3]) + np.testing.assert_array_equal(updated.loc[stock_name], [1.2, 1.3]) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "filename", ["uprating_factors.csv", "uprating_growth_factors.csv"] +) +@pytest.mark.parametrize("purchase_name,stock_name", PROPERTY_INPUTS) +def test_purchase_uprating_matches_stock(filename, purchase_name, stock_name): + factors = pd.read_csv(STORAGE_FOLDER / filename, index_col="Variable") + np.testing.assert_array_equal(factors.loc[purchase_name], factors.loc[stock_name]) + + +@dataclass +class _SyntheticDataset: + household: pd.DataFrame + time_period: int + + @property + def tables(self): + return [self.household] + + def copy(self): + return _SyntheticDataset(self.household.copy(), self.time_period) + + +@pytest.mark.unit +@pytest.mark.parametrize("purchase_name,stock_name", PROPERTY_INPUTS) +def test_base_year_materialisation_preserves_purchase_stock_cap( + purchase_name, stock_name +): + stock = np.array([50_000.0, 250_000.0, 1_000_000.0]) + dataset = _SyntheticDataset( + pd.DataFrame({stock_name: stock, purchase_name: stock}), time_period=2025 + ) + saved = uprate_dataset(dataset, 2024) + assert saved.time_period == 2024 + assert np.all(saved.household[purchase_name] <= saved.household[stock_name]) + assert np.all(saved.household[purchase_name] < dataset.household[purchase_name]) + restored = uprate_dataset(saved, 2025) + np.testing.assert_allclose(restored.household[purchase_name], stock) diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index ff3939fb..1349778e 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -1,6 +1,13 @@ -from policyengine_uk_data.storage import STORAGE_FOLDER +from __future__ import annotations + +from typing import TYPE_CHECKING + import pandas as pd -from policyengine_uk.data import UKSingleYearDataset + +from policyengine_uk_data.storage import STORAGE_FOLDER + +if TYPE_CHECKING: + from policyengine_uk.data import UKSingleYearDataset START_YEAR = 2020 END_YEAR = 2034 @@ -86,11 +93,13 @@ def create_policyengine_uprating_factors_table(): # population indices would inflate final calibrated population above the # existing fidelity guard. df = _apply_household_weight_uprating_override(df) + df = _apply_property_purchase_uprating_override(df) # Ensure petrol/diesel use sourced road-fuel clearances and model pump # prices. This keeps litres aligned after PolicyEngine divides by price. df = _apply_road_fuel_litre_proxy_override(df) + df = df.sort_index() df.to_csv(STORAGE_FOLDER / "uprating_factors.csv") # Create a table with growth factors by year @@ -104,6 +113,24 @@ def create_policyengine_uprating_factors_table(): return df +def _apply_property_purchase_uprating_override(df: pd.DataFrame) -> pd.DataFrame: + """Keep annual purchase prices on the same index as their stock caps. + + Also supplies these rows when building with policyengine-uk versions + predating #2241, whose purchase inputs had no uprating index. + """ + for purchase, stock in ( + ( + "additional_residential_property_purchased", + "other_residential_property_value", + ), + ("non_residential_property_purchased", "non_residential_property_value"), + ): + if stock in df.index: + df.loc[purchase] = df.loc[stock] + return df + + def _apply_household_weight_uprating_override(df: pd.DataFrame) -> pd.DataFrame: """Restore the household-weight index used by the calibrated dataset.""" if "household_weight" not in df.index: diff --git a/scripts/validate_property_purchases.py b/scripts/validate_property_purchases.py new file mode 100644 index 00000000..6e715936 --- /dev/null +++ b/scripts/validate_property_purchases.py @@ -0,0 +1,184 @@ +"""Validate purchase imputation on a local enhanced FRS, printing aggregates only. + +Run from the repository root with the desired policyengine-uk on PYTHONPATH:: + + python scripts/validate_property_purchases.py /path/enhanced_frs_2023_24.h5 \ + --years 2023 2024 2025 + +Only four household columns are read. No dataset is downloaded or written. +Later years uprate stocks and weights with the model's dataset indices before +re-imputing purchases against that year's assumptions. This is a validation +of annual imputation, rather than a rebuilt or recalibrated dataset. +""" + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +# Make direct execution from scripts/ resolve this checkout. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from policyengine_uk import parameters as model_parameters + +from policyengine_uk_data.datasets.imputations.property_purchases import ( + SDLT_REGIONS, + impute_property_purchase_values, +) +from policyengine_uk_data.parameters import load_parameter + +PROPERTY_STOCKS = { + "additional_residential": "other_residential_property_value", + "non_residential": "non_residential_property_value", +} +STOCK_INDEX = "gov.economic_assumptions.yoy_growth.obr.per_capita.gdp" +WEIGHT_INDEX = "gov.economic_assumptions.yoy_growth.ons.population" + + +def growth_factor(parameters, index, source_year, year): + """Use the same year-on-year factors as dataset projection.""" + return float( + np.prod( + [ + 1 + parameters.get_child(index)(str(y)) + for y in range(source_year + 1, year + 1) + ] + ) + ) + + +def validate(household, source_year, year, parameters): + """Return weighted E+NI aggregates and checks; never return record values.""" + if year < source_year: + raise ValueError("Validation years must not precede the dataset year") + stock_factor = growth_factor(parameters, STOCK_INDEX, source_year, year) + weight_factor = growth_factor(parameters, WEIGHT_INDEX, source_year, year) + weights = household.household_weight.to_numpy(dtype=float) * weight_factor + sdlt_mask = household.region.astype(str).isin(SDLT_REGIONS).to_numpy() + stamp_duty = parameters(str(year)).gov.hmrc.stamp_duty + rows = [] + for property_type, stock_name in PROPERTY_STOCKS.items(): + assumptions = load_parameter( + "stochastic", "additional_property_purchases", year + )[property_type] + stock = household[stock_name].to_numpy(dtype=float) * stock_factor + prices = impute_property_purchase_values( + stock, weights, sdlt_mask, year, property_type + ) + repeat = impute_property_purchase_values( + stock, weights, sdlt_mask, year, property_type + ) + purchased = prices > 0 + minimum = assumptions["minimum_price"] + eligible = (stock >= minimum) & (weights > 0) + fitted_weights = weights * (purchased & sdlt_mask) + transactions = float(fitted_weights.sum()) + value = float(np.dot(prices, weights * sdlt_mask)) + checks = { + "reproducible": bool(np.array_equal(prices, repeat)), + "owners_only": bool(np.all(~purchased | eligible)), + "price_bounds": bool(np.all(prices <= stock)), + "minimum_price": bool(np.all(prices[purchased] >= minimum)), + "transactions_within_5_percent": bool( + abs(transactions / assumptions["transactions"] - 1) <= 0.05 + ), + "value_within_5_percent": bool( + abs(value / assumptions["value"] - 1) <= 0.05 + ), + } + if not all(checks.values()): + raise ValueError(f"Aggregate validation failed for {year} {property_type}") + if property_type == "additional_residential": + purchase = stamp_duty.residential.purchase.additional + liability = purchase.rate.calc(prices) * (prices >= purchase.min) + else: + liability = stamp_duty.non_residential.purchase.calc(prices) + bands = assumptions["price_bands"] + counts = np.array([band["transactions"] for band in bands], dtype=float) + means = np.array([band["value"] for band in bands]) / counts + raw_prices = np.random.default_rng(assumptions["price_seed"]).choice( + means, size=len(stock), p=counts / counts.sum() + ) + uncapped = purchased & (prices < stock) & (prices > minimum) + price_scale = ( + float(np.median(prices[uncapped] / raw_prices[uncapped])) + if uncapped.any() + else None + ) + rows.append( + { + "year": year, + "property_type": property_type, + "target_transactions": assumptions["transactions"], + "transactions": transactions, + "target_value": assumptions["value"], + "value": value, + "reference_gross_receipts": assumptions["revenue"], + "model_gross_sdlt": float(np.dot(liability, weights * sdlt_mask)), + "two_percentage_point_hrad_exposure": ( + 0.02 * value if property_type == "additional_residential" else None + ), + "eligible_weighted_owners": float( + np.dot(eligible, weights * sdlt_mask) + ), + "eligible_mean_stock": float( + np.dot(stock, weights * (eligible & sdlt_mask)) + / np.dot(eligible, weights * sdlt_mask) + ), + "selected_stock_capacity": float(np.dot(stock, fitted_weights)), + "weighted_purchase_rate": float( + transactions / np.dot(eligible, weights * sdlt_mask) + ), + "price_band_count": len(bands), + "price_scale": price_scale, + "weighted_stock_capped_share": float( + np.dot(purchased & (prices == stock), weights * sdlt_mask) + / transactions + ), + "stock_uprating_factor": stock_factor, + "weight_uprating_factor": weight_factor, + "checks": checks, + } + ) + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dataset", type=Path) + parser.add_argument("--years", type=int, nargs="+") + args = parser.parse_args() + source_year = int(pd.read_hdf(args.dataset, key="time_period").iloc[0]) + household = pd.read_hdf( + args.dataset, + key="household", + columns=["household_weight", "region", *PROPERTY_STOCKS.values()], + ) + rows = [ + row + for year in args.years or [source_year] + for row in validate(household, source_year, year, model_parameters) + ] + print( + json.dumps( + { + "source_year": source_year, + "scope": "England and Northern Ireland; weighted aggregates only", + "receipts_note": ( + "Model gross SDLT uses the policyengine-uk 1 January higher-rates " + "scale (the full tax, including standard rates). HMRC receipts " + "cover a financial year, policy changes and replacement-home " + "refund timing, so they are reference benchmarks, not fitted targets." + ), + "results": rows, + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main()