From 436ce70accf2094d47d33e654b8277e727ea868b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:45:24 -0400 Subject: [PATCH 1/6] Impute seeded household additional-property purchases Fit explicit HMRC England and Northern Ireland count and value assumptions after wealth and geography exist, with independent purchase draws and hard stock caps. Co-Authored-By: Claude Opus 5.5 --- changelog.d/556.added.md | 1 + .../datasets/create_datasets.py | 10 +- .../datasets/imputations/__init__.py | 1 + .../imputations/property_purchases.py | 128 ++++++++++++++++++ .../additional_property_purchases.yaml | 82 +++++++++++ 5 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 changelog.d/556.added.md create mode 100644 policyengine_uk_data/datasets/imputations/property_purchases.py create mode 100644 policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml diff --git a/changelog.d/556.added.md b/changelog.d/556.added.md new file mode 100644 index 000000000..d33da5520 --- /dev/null +++ b/changelog.d/556.added.md @@ -0,0 +1 @@ +Impute seeded additional-dwelling and non-residential purchase prices from household property stocks, fitting explicit HMRC household transaction and value assumptions with a hard stock cap. diff --git a/policyengine_uk_data/datasets/create_datasets.py b/policyengine_uk_data/datasets/create_datasets.py index 310081839..32ab57684 100644 --- a/policyengine_uk_data/datasets/create_datasets.py +++ b/policyengine_uk_data/datasets/create_datasets.py @@ -111,6 +111,7 @@ def main(): "Impute student loan plan", "Assign Pension Credit take-up", "Clone and assign OA geography", + "Impute property purchases", "Calibrate constituency weights", "Calibrate local authority weights", "Export long local geography weights", @@ -163,6 +164,7 @@ def main(): impute_salary_sacrifice, impute_student_loan_plan, uprate_property_by_region, + impute_property_purchases, ) # Apply imputations with progress tracking @@ -247,6 +249,12 @@ def main(): ) update_dataset(materialize_calibration_step, "completed") + # Stocks and final household geography now exist at the calibration + # year. Supply independent purchase inputs before fitting SDLT. + update_dataset("Impute property purchases", "processing") + frs = impute_property_purchases(frs) + update_dataset("Impute property purchases", "completed") + # Calibrate constituency weights with nested progress update_dataset("Calibrate constituency weights", "processing") @@ -417,7 +425,7 @@ def main(): "tiny_base_dataset": frs_release.tiny_base_dataset_file, "tiny_enhanced_dataset": frs_release.tiny_enhanced_dataset_file, "long_geography_weights": "local_geography_weights.csv.gz", - "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan", + "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan, property_purchases", "calibration": "national, LA and constituency targets", "pension_credit_take_up": pension_credit_takeup, }, diff --git a/policyengine_uk_data/datasets/imputations/__init__.py b/policyengine_uk_data/datasets/imputations/__init__.py index 736941f1d..ba75f56b2 100644 --- a/policyengine_uk_data/datasets/imputations/__init__.py +++ b/policyengine_uk_data/datasets/imputations/__init__.py @@ -8,3 +8,4 @@ from .salary_sacrifice import impute_salary_sacrifice from .student_loans import impute_student_loan_plan from .regional_property_uprating import uprate_property_by_region +from .property_purchases import impute_property_purchases diff --git a/policyengine_uk_data/datasets/imputations/property_purchases.py b/policyengine_uk_data/datasets/imputations/property_purchases.py new file mode 100644 index 000000000..61f86a2d8 --- /dev/null +++ b/policyengine_uk_data/datasets/imputations/property_purchases.py @@ -0,0 +1,128 @@ +"""Impute one annual purchase from each kind of household property stock. + +HMRC targets cover England and Northern Ireland. Independent seeded event +draws are calibrated there; the resulting rates also apply in Scotland and +Wales. Prices are separate seeded draws, never the household's entire stock +by default. Assumptions and sources live in stochastic parameters. +""" + +import numpy as np + +from policyengine_uk_data.parameters import load_parameter + + +def impute_property_purchase_values( + stock: np.ndarray, + weights: np.ndarray, + sdlt_mask: np.ndarray, + year: int, + property_type: str, +) -> np.ndarray: + """Return seeded purchase prices, fitting weighted events and capped value. + + The closest attainable weighted count determines the Bernoulli threshold. + The value target is fitted by multiplying sampled band-average prices, + with the stock cap applied throughout. Zero-weight donors never purchase. + """ + stock = np.asarray(stock, dtype=float) + weights = np.asarray(weights, dtype=float) + sdlt_mask = np.asarray(sdlt_mask, dtype=bool) + if ( + stock.ndim != 1 + or stock.shape != weights.shape + or stock.shape != sdlt_mask.shape + ): + raise ValueError( + "Property stocks, weights and SDLT mask must be aligned vectors" + ) + if ( + not np.all(np.isfinite(stock)) + or not np.all(np.isfinite(weights)) + or np.any(stock < 0) + or np.any(weights < 0) + ): + raise ValueError("Property stocks and weights must be finite and non-negative") + + assumptions = load_parameter("stochastic", "additional_property_purchases", year)[ + property_type + ] + target_count = assumptions["transactions"] + target_value = assumptions["value"] + minimum_price = assumptions["minimum_price"] + owners = (stock >= minimum_price) & (weights > 0) + eligible = np.flatnonzero(owners & sdlt_mask) + if weights[eligible].sum() < target_count: + raise ValueError("Insufficient weighted property owners for purchase target") + + events = np.random.default_rng(assumptions["purchase_seed"]).random(len(stock)) + ranked = eligible[np.argsort(events[eligible], kind="stable")] + counts = np.concatenate(([0.0], np.cumsum(weights[ranked]))) + n_purchases = int(np.argmin(np.abs(counts - target_count))) + if n_purchases == len(ranked): + threshold = 1.0 + elif n_purchases == 0: + threshold = 0.0 + else: + threshold = (events[ranked[n_purchases - 1]] + events[ranked[n_purchases]]) / 2 + purchased = owners & (events < threshold) + fitted_weights = weights * (purchased & sdlt_mask) + capacity = np.dot(stock, fitted_weights) + if capacity < target_value: + raise ValueError( + "Selected property stock cannot support the purchase value target" + ) + if minimum_price * fitted_weights.sum() > target_value: + raise ValueError("Purchase value target is below the minimum transaction price") + + bands = assumptions["price_bands"] + band_counts = np.array([band["transactions"] for band in bands], dtype=float) + band_prices = np.array([band["value"] for band in bands]) / band_counts + price_rng = np.random.default_rng(assumptions["price_seed"]) + prices = price_rng.choice( + band_prices, size=len(stock), p=band_counts / band_counts.sum() + ) + + def capped_prices(scale): + return np.minimum(stock, np.maximum(minimum_price, prices * scale)) + + def weighted_value(scale): + return np.dot(capped_prices(scale), fitted_weights) + + lower, upper = 0.0, 1.0 + while weighted_value(upper) < target_value: + upper *= 2 + for _ in range(80): + midpoint = (lower + upper) / 2 + if weighted_value(midpoint) < target_value: + lower = midpoint + else: + upper = midpoint + return np.where(purchased, capped_prices(upper), 0.0) + + +def impute_property_purchases(dataset, year=None): + """Add annual purchase inputs after wealth imputation, before calibration.""" + dataset = dataset.copy() + year = int(dataset.time_period) if year is None else year + household = dataset.household + sdlt_mask = ~household["region"].astype(str).isin(["SCOTLAND", "WALES"]).values + for property_type, stock_name, purchase_name in ( + ( + "additional_residential", + "other_residential_property_value", + "additional_residential_property_purchased", + ), + ( + "non_residential", + "non_residential_property_value", + "non_residential_property_purchased", + ), + ): + household[purchase_name] = impute_property_purchase_values( + household[stock_name].values, + household["household_weight"].values, + sdlt_mask, + year, + property_type, + ) + return dataset diff --git a/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml new file mode 100644 index 000000000..e4a477c9c --- /dev/null +++ b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml @@ -0,0 +1,82 @@ +description: Household additional-dwelling and non-residential purchase assumptions +metadata: + label: Additional property purchases + reference: + - title: HMRC Annual Stamp Tax Statistics 2024 to 2025, Tables 6b, 8a and 8b + href: https://www.gov.uk/government/statistics/uk-stamp-tax-statistics + - title: HMRC Annual Stamp Tax Statistics 2024 to 2025, corrected tables + href: https://assets.publishing.service.gov.uk/media/69aef72fc78869bf8eb8a589/Annual_Stamps_2024_25_correction.ods + +# HMRC SDLT tables cover England and Northern Ireland (E+NI), transactions +# of at least GBP 40,000, and financial years ending in the following March. +# Transaction counts are rounded to 100; values and receipts to GBP 5 million. +# All values below are pounds. Imputation fits E+NI weighted counts and values. +# The purchase rate is target transactions / eligible E+NI weighted owners of +# property stock of at least GBP 40,000; a seeded uniform threshold is selected +# to give the closest attainable weighted count. The same threshold applies to +# owners elsewhere in the UK, an explicit extrapolation without devolved targets. +# Prices start from a seeded draw among band means (band value / transactions). +# A common scale then fits the weighted purchase value with price equal to +# min(stock, max(40,000, scale * draw)), so prices stay within HMRC's +# transaction coverage and never exceed the owner's stock. +# This is a capped, calibrated band-mean method, not an exact draw from the +# price-band distribution conditional on stock. Rounded band counts are +# normalised by their sum rather than by the headline count. +# +# Household HRAD is not separately published. The modelling choice is neutral +# allocation using individuals' shares of all residential buyers in Table 8a: +# transactions = 211,700 * 925,100 / (925,100 + 124,500) = 186,588.8624; +# value = 77,970,000,000 * 335,515 / (335,515 + 44,690) = 68,805,261,766.6785. +# Revenue is a reference benchmark, not fitted by imputation: +# 5,435,000,000 * 9,145 / (9,145 + 1,230) = 4,790,657,831.3253. +# This allocation is an assumption, not an observed household HRAD series. +# Table 6b reports all-buyer gross HRAD SDLT before replacement-home refunds; +# its surcharge-only receipts are GBP 2.790bn. The independent household lower +# bounds implied by Tables 6b/8a are 87,200 transactions, GBP 33.280bn value, +# and GBP 4.205bn gross receipts. They are bounds, not the selected targets. +# For review, 2024-25 household HRAD targets and price bands are held flat across +# survey/calibration years: there is no annual official household HRAD series +# and this assumption is not a forecast. Non-residential targets use the +# directly observed Table 8b individuals rows for 2023-24 and 2024-25, replacing +# the older household rows in policyengine-uk's stamp_duty/statistics.yaml. +values: + 2013-01-01: + additional_residential: &additional_residential + transactions: 186588.8624237805 + value: 68805261766.6785 + revenue: 4790657831.325301 + minimum_price: 40000 + purchase_seed: 55601 + price_seed: 55602 + # Table 6b 2024-25: last band combines all five published bands over GBP 2m. + # Band sums differ slightly from headline totals because of rounding. + price_bands: + - {transactions: 113500, value: 16635000000} # Up to GBP 250,000 + - {transactions: 60700, value: 21510000000} # GBP 250,001-500,000 + - {transactions: 27500, value: 18770000000} # GBP 500,001-1,000,000 + - {transactions: 5300, value: 6545000000} # GBP 1,000,001-1,500,000 + - {transactions: 2000, value: 3515000000} # GBP 1,500,001-2,000,000 + - {transactions: 2600, value: 10990000000} # Over GBP 2,000,000 + non_residential: + # 2023-24 Table 8b individuals: 24,800 / GBP 6.955bn / GBP 230m. + # One representative mean price is used before stock-capped scaling. + transactions: 24800 + value: 6955000000 + revenue: 230000000 + minimum_price: 40000 + purchase_seed: 55603 + price_seed: 55604 + price_bands: + - {transactions: 24800, value: 6955000000} + 2024-01-01: + additional_residential: *additional_residential + non_residential: + # 2024-25 Table 8b individuals: 25,000 / GBP 7.640bn / GBP 245m. + transactions: 25000 + value: 7640000000 + revenue: 245000000 + minimum_price: 40000 + purchase_seed: 55603 + price_seed: 55604 + price_bands: + - {transactions: 25000, value: 7640000000} From 4fc22dcf5b0c355140f52c2c8ef3a113032e9111 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 20:02:28 -0400 Subject: [PATCH 2/6] Use stock-proportional purchase hazard to support HMRC values Keep the seeded transaction draw calibrated while allowing larger property portfolios to trade more often. Document this explicit modelling assumption rather than tune seeds to a particular dataset. Co-Authored-By: Claude Opus 5.5 --- .../imputations/property_purchases.py | 27 ++++++++++++++++--- .../additional_property_purchases.yaml | 17 +++++++++--- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/policyengine_uk_data/datasets/imputations/property_purchases.py b/policyengine_uk_data/datasets/imputations/property_purchases.py index 61f86a2d8..9f7327221 100644 --- a/policyengine_uk_data/datasets/imputations/property_purchases.py +++ b/policyengine_uk_data/datasets/imputations/property_purchases.py @@ -10,6 +10,19 @@ from policyengine_uk_data.parameters import load_parameter +SDLT_REGIONS = ( + "NORTH_EAST", + "NORTH_WEST", + "YORKSHIRE", + "EAST_MIDLANDS", + "WEST_MIDLANDS", + "EAST_OF_ENGLAND", + "LONDON", + "SOUTH_EAST", + "SOUTH_WEST", + "NORTHERN_IRELAND", +) + def impute_property_purchase_values( stock: np.ndarray, @@ -20,7 +33,8 @@ def impute_property_purchase_values( ) -> np.ndarray: """Return seeded purchase prices, fitting weighted events and capped value. - The closest attainable weighted count determines the Bernoulli threshold. + Purchase hazard is proportional to stock, an explicit portfolio-turnover + assumption. The closest weighted count determines the event threshold. The value target is fitted by multiplying sampled band-average prices, with the stock cap applied throughout. Zero-weight donors never purchase. """ @@ -54,12 +68,17 @@ def impute_property_purchase_values( if weights[eligible].sum() < target_count: raise ValueError("Insufficient weighted property owners for purchase target") - events = np.random.default_rng(assumptions["purchase_seed"]).random(len(stock)) + reference_stock = np.average(stock[eligible], weights=weights[eligible]) + hazard = (np.maximum(stock, minimum_price) / reference_stock) ** assumptions[ + "purchase_stock_exponent" + ] + uniform = np.random.default_rng(assumptions["purchase_seed"]).random(len(stock)) + events = -np.log1p(-uniform) / hazard ranked = eligible[np.argsort(events[eligible], kind="stable")] counts = np.concatenate(([0.0], np.cumsum(weights[ranked]))) n_purchases = int(np.argmin(np.abs(counts - target_count))) if n_purchases == len(ranked): - threshold = 1.0 + threshold = np.inf elif n_purchases == 0: threshold = 0.0 else: @@ -105,7 +124,7 @@ def impute_property_purchases(dataset, year=None): dataset = dataset.copy() year = int(dataset.time_period) if year is None else year household = dataset.household - sdlt_mask = ~household["region"].astype(str).isin(["SCOTLAND", "WALES"]).values + sdlt_mask = household["region"].astype(str).isin(SDLT_REGIONS).values for property_type, stock_name, purchase_name in ( ( "additional_residential", diff --git a/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml index e4a477c9c..6a7715ce9 100644 --- a/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml +++ b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml @@ -11,10 +11,16 @@ metadata: # of at least GBP 40,000, and financial years ending in the following March. # Transaction counts are rounded to 100; values and receipts to GBP 5 million. # All values below are pounds. Imputation fits E+NI weighted counts and values. -# The purchase rate is target transactions / eligible E+NI weighted owners of -# property stock of at least GBP 40,000; a seeded uniform threshold is selected -# to give the closest attainable weighted count. The same threshold applies to -# owners elsewhere in the UK, an explicit extrapolation without devolved targets. +# Mean annual purchase rate is target transactions / eligible E+NI weighted +# owners of stock of at least GBP 40,000. Stock-proportional purchase hazard +# (exponent 1) assumes larger portfolios are more likely to trade each year; +# it is a modelling choice, not an observed household transaction propensity. +# Independent seeded exponential event scores divided by relative stock +# select the closest attainable weighted count. The same hazard and threshold +# apply elsewhere in the UK, an extrapolation without devolved targets. +# Uniform owner selection on the local enhanced FRS 2023-24 selected too little +# stock to support the assumed HRAD value. Stock weighting addresses that +# feasibility issue without tuning seeds to a particular dataset. # Prices start from a seeded draw among band means (band value / transactions). # A common scale then fits the weighted purchase value with price equal to # min(stock, max(40,000, scale * draw)), so prices stay within HMRC's @@ -46,6 +52,7 @@ values: value: 68805261766.6785 revenue: 4790657831.325301 minimum_price: 40000 + purchase_stock_exponent: 1 purchase_seed: 55601 price_seed: 55602 # Table 6b 2024-25: last band combines all five published bands over GBP 2m. @@ -64,6 +71,7 @@ values: value: 6955000000 revenue: 230000000 minimum_price: 40000 + purchase_stock_exponent: 1 purchase_seed: 55603 price_seed: 55604 price_bands: @@ -76,6 +84,7 @@ values: value: 7640000000 revenue: 245000000 minimum_price: 40000 + purchase_stock_exponent: 1 purchase_seed: 55603 price_seed: 55604 price_bands: From 9e5554cf57d8afe2863b02f03fcbbda23be45d66 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 20:02:59 -0400 Subject: [PATCH 3/6] Preserve purchase stock caps across dataset uprating Give both purchase inputs the same index as their respective stocks, including when generating tables with model versions predating #2241. Cover missing and stale indices and calibration-to-base-year materialisation. Co-Authored-By: Claude Opus 5.5 --- .../storage/uprating_factors.csv | 2 + .../storage/uprating_growth_factors.csv | 2 + .../test_uprating_property_purchase_inputs.py | 72 +++++++++++++++++++ policyengine_uk_data/utils/uprating.py | 20 ++++++ 4 files changed, 96 insertions(+) create mode 100644 policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 149ef19ff..5e98e38be 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -1,4 +1,5 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 +additional_residential_property_purchased,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 afcs_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 alcohol_and_tobacco_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 benunit_rent,1.0,1.0,1.0,1.11,1.184,1.223,1.275,1.312,1.351,1.392,1.392,1.392,1.392,1.392,1.392 @@ -49,6 +50,7 @@ miscellaneous_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384 mortgage_capital_repayment,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 mortgage_interest_repayment,1.0,1.0,1.262,1.874,2.288,2.599,2.927,3.167,3.3,3.455,3.455,3.455,3.455,3.455,3.455 net_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +non_residential_property_purchased,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 non_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 956f740d7..9384e9379 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -1,4 +1,5 @@ Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034 +additional_residential_property_purchased,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 afcs_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 alcohol_and_tobacco_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 benunit_rent,0,0.0,0.0,0.11,0.067,0.033,0.043,0.029,0.03,0.03,0.0,0.0,0.0,0.0,0.0 @@ -49,6 +50,7 @@ miscellaneous_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0. mortgage_capital_repayment,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 mortgage_interest_repayment,0,0.0,0.262,0.485,0.221,0.136,0.126,0.082,0.042,0.047,0.0,0.0,0.0,0.0,0.0 net_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +non_residential_property_purchased,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 non_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py b/policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py new file mode 100644 index 000000000..972754e16 --- /dev/null +++ b/policyengine_uk_data/tests/test_uprating_property_purchase_inputs.py @@ -0,0 +1,72 @@ +"""Purchase inputs must retain their stock caps during year materialisation.""" + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.storage import STORAGE_FOLDER +from policyengine_uk_data.utils.uprating import ( + _apply_property_purchase_uprating_override, + uprate_dataset, +) + +PROPERTY_INPUTS = ( + ("additional_residential_property_purchased", "other_residential_property_value"), + ("non_residential_property_purchased", "non_residential_property_value"), +) + + +@pytest.mark.unit +@pytest.mark.parametrize("purchase_name,stock_name", PROPERTY_INPUTS) +@pytest.mark.parametrize("purchase_exists", [False, True]) +def test_uprating_generator_adds_or_replaces_purchase_index( + purchase_name, stock_name, purchase_exists +): + factors = pd.DataFrame([[1.2, 1.3]], index=[stock_name], columns=[2024, 2025]) + if purchase_exists: + factors.loc[purchase_name] = [9.0, 10.0] + updated = _apply_property_purchase_uprating_override(factors) + np.testing.assert_array_equal(updated.loc[purchase_name], [1.2, 1.3]) + np.testing.assert_array_equal(updated.loc[stock_name], [1.2, 1.3]) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "filename", ["uprating_factors.csv", "uprating_growth_factors.csv"] +) +@pytest.mark.parametrize("purchase_name,stock_name", PROPERTY_INPUTS) +def test_purchase_uprating_matches_stock(filename, purchase_name, stock_name): + factors = pd.read_csv(STORAGE_FOLDER / filename, index_col="Variable") + np.testing.assert_array_equal(factors.loc[purchase_name], factors.loc[stock_name]) + + +@dataclass +class _SyntheticDataset: + household: pd.DataFrame + time_period: int + + @property + def tables(self): + return [self.household] + + def copy(self): + return _SyntheticDataset(self.household.copy(), self.time_period) + + +@pytest.mark.unit +@pytest.mark.parametrize("purchase_name,stock_name", PROPERTY_INPUTS) +def test_base_year_materialisation_preserves_purchase_stock_cap( + purchase_name, stock_name +): + stock = np.array([50_000.0, 250_000.0, 1_000_000.0]) + dataset = _SyntheticDataset( + pd.DataFrame({stock_name: stock, purchase_name: stock}), time_period=2025 + ) + saved = uprate_dataset(dataset, 2024) + assert saved.time_period == 2024 + assert np.all(saved.household[purchase_name] <= saved.household[stock_name]) + assert np.all(saved.household[purchase_name] < dataset.household[purchase_name]) + restored = uprate_dataset(saved, 2025) + np.testing.assert_allclose(restored.household[purchase_name], stock) diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index ff3939fb1..53c7e4015 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -86,11 +86,13 @@ def create_policyengine_uprating_factors_table(): # population indices would inflate final calibrated population above the # existing fidelity guard. df = _apply_household_weight_uprating_override(df) + df = _apply_property_purchase_uprating_override(df) # Ensure petrol/diesel use sourced road-fuel clearances and model pump # prices. This keeps litres aligned after PolicyEngine divides by price. df = _apply_road_fuel_litre_proxy_override(df) + df = df.sort_index() df.to_csv(STORAGE_FOLDER / "uprating_factors.csv") # Create a table with growth factors by year @@ -104,6 +106,24 @@ def create_policyengine_uprating_factors_table(): return df +def _apply_property_purchase_uprating_override(df: pd.DataFrame) -> pd.DataFrame: + """Keep annual purchase prices on the same index as their stock caps. + + Also supplies these rows when building with policyengine-uk versions + predating #2241, whose purchase inputs had no uprating index. + """ + for purchase, stock in ( + ( + "additional_residential_property_purchased", + "other_residential_property_value", + ), + ("non_residential_property_purchased", "non_residential_property_value"), + ): + if stock in df.index: + df.loc[purchase] = df.loc[stock] + return df + + def _apply_household_weight_uprating_override(df: pd.DataFrame) -> pd.DataFrame: """Restore the household-weight index used by the calibrated dataset.""" if "household_weight" not in df.index: From e000c1ce21247b57ab8b3034cbad87f3720788bd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 20:07:36 -0400 Subject: [PATCH 4/6] Use HMRC individuals price bands for non-residential purchases Replace the single average price with Table 8b household price-band draws before stock-capped value fitting, retaining observed count and value targets. Co-Authored-By: Claude Opus 5.5 --- .../additional_property_purchases.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml index 6a7715ce9..c6671b709 100644 --- a/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml +++ b/policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml @@ -45,6 +45,9 @@ metadata: # and this assumption is not a forecast. Non-residential targets use the # directly observed Table 8b individuals rows for 2023-24 and 2024-25, replacing # the older household rows in policyengine-uk's stamp_duty/statistics.yaml. +# Their price bands also use individuals only. The 2024-25 band counts sum to +# 25,100 rather than the 25,000 headline because of rounding; probabilities +# use that band sum. Stock caps and the fitted scale modify the band distribution. values: 2013-01-01: additional_residential: &additional_residential @@ -66,7 +69,7 @@ values: - {transactions: 2600, value: 10990000000} # Over GBP 2,000,000 non_residential: # 2023-24 Table 8b individuals: 24,800 / GBP 6.955bn / GBP 230m. - # One representative mean price is used before stock-capped scaling. + # Table 8b individuals' price-band means, before stock-capped scaling. transactions: 24800 value: 6955000000 revenue: 230000000 @@ -75,7 +78,12 @@ values: purchase_seed: 55603 price_seed: 55604 price_bands: - - {transactions: 24800, value: 6955000000} + - {transactions: 18800, value: 1545000000} # Up to GBP 250,000 + - {transactions: 3300, value: 1150000000} # GBP 250,001-500,000 + - {transactions: 1500, value: 1085000000} # GBP 500,001-1,000,000 + - {transactions: 500, value: 640000000} # GBP 1,000,001-1,500,000 + - {transactions: 200, value: 415000000} # GBP 1,500,001-2,000,000 + - {transactions: 500, value: 2120000000} # Over GBP 2,000,000 2024-01-01: additional_residential: *additional_residential non_residential: @@ -88,4 +96,9 @@ values: purchase_seed: 55603 price_seed: 55604 price_bands: - - {transactions: 25000, value: 7640000000} + - {transactions: 18800, value: 1560000000} # Up to GBP 250,000 + - {transactions: 3400, value: 1210000000} # GBP 250,001-500,000 + - {transactions: 1600, value: 1100000000} # GBP 500,001-1,000,000 + - {transactions: 500, value: 650000000} # GBP 1,000,001-1,500,000 + - {transactions: 300, value: 445000000} # GBP 1,500,001-2,000,000 + - {transactions: 500, value: 2675000000} # Over GBP 2,000,000 From e99792644081ad2af53c699b049e9264556e4bf7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 20:16:44 -0400 Subject: [PATCH 5/6] Test and validate household property purchase imputation Cover seeded reproducibility, ownership, stock and minimum-price bounds, weighted targets, saved inputs and higher-rates SDLT. Add an aggregate-only local enhanced FRS validation command. Co-Authored-By: Claude Opus 5.5 --- .../test_additional_property_purchased.py | 299 ++++++++++++++++++ scripts/validate_property_purchases.py | 181 +++++++++++ 2 files changed, 480 insertions(+) create mode 100644 policyengine_uk_data/tests/test_additional_property_purchased.py create mode 100644 scripts/validate_property_purchases.py diff --git a/policyengine_uk_data/tests/test_additional_property_purchased.py b/policyengine_uk_data/tests/test_additional_property_purchased.py new file mode 100644 index 000000000..f0c4df83b --- /dev/null +++ b/policyengine_uk_data/tests/test_additional_property_purchased.py @@ -0,0 +1,299 @@ +"""Additional-property purchases are reproducible, stock capped and calibrated. + +Synthetic counts tolerate 10% error and value fits have relative tolerance +1e-8. Saved enhanced FRS aggregates tolerate 20% after weight calibration. +Dataset assertions report only aggregate diagnostics, never household rows. +""" + +from dataclasses import dataclass + +import numpy as np +import pandas as pd +import pytest + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.datasets.imputations.property_purchases import ( + impute_property_purchase_values, + impute_property_purchases, +) +from policyengine_uk_data.parameters import load_parameter + + +PROPERTY_TYPES = ( + ( + "additional_residential", + "other_residential_property_value", + "additional_residential_property_purchased", + ), + ( + "non_residential", + "non_residential_property_value", + "non_residential_property_purchased", + ), +) +PROPERTY_TYPE_NAMES = tuple(item[0] for item in PROPERTY_TYPES) +RELEASE_YEARS = sorted( + {CURRENT_FRS_RELEASE.base_year, CURRENT_FRS_RELEASE.calibration_year} +) +SYNTHETIC_YEARS = sorted({2023, *RELEASE_YEARS}) + + +@pytest.fixture +def synthetic_households(): + """Heterogeneous owners, weights and nations with ample purchase capacity.""" + stock = np.geomspace(50_000.0, 50_000_000.0, 4_000) + stock[::5] = 0.0 + stock[1::41] = 10_000.0 + weights = np.linspace(100.0, 500.0, len(stock)) + weights[::11] = 0.0 + sdlt_mask = np.arange(len(stock)) % 7 != 0 + return stock, weights, sdlt_mask + + +def _targets(property_type, year): + return load_parameter("stochastic", "additional_property_purchases", year)[ + property_type + ] + + +def _assert_aggregate_targets( + prices, weights, sdlt_mask, targets, tolerance, value_tolerance=None +): + value_tolerance = tolerance if value_tolerance is None else value_tolerance + transactions = float(np.sum(weights[(prices > 0) & sdlt_mask])) + value = float(np.dot(prices[sdlt_mask], weights[sdlt_mask])) + assert transactions == pytest.approx(targets["transactions"], rel=tolerance), ( + f"Weighted transactions {transactions:,.0f}; " + f"target {targets['transactions']:,.0f}; tolerance {tolerance:.0%}" + ) + assert value == pytest.approx(targets["value"], rel=value_tolerance), ( + f"Weighted purchase value £{value / 1e9:.3f}bn; " + f"target £{targets['value'] / 1e9:.3f}bn; " + f"relative tolerance {value_tolerance:g}" + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +def test_purchase_imputation_is_independent_of_global_rng( + synthetic_households, property_type +): + stock, weights, sdlt_mask = synthetic_households + state = np.random.get_state() + try: + np.random.seed(1) + first = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, property_type + ) + np.random.seed(999) + np.random.random(10_000) + second = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, property_type + ) + finally: + np.random.set_state(state) + np.testing.assert_array_equal(first, second) + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +@pytest.mark.parametrize("year", SYNTHETIC_YEARS) +def test_purchase_prices_and_weighted_targets( + synthetic_households, property_type, year +): + stock, weights, sdlt_mask = synthetic_households + prices = impute_property_purchase_values( + stock, weights, sdlt_mask, year, property_type + ) + assert np.all(np.isfinite(prices)) + assert np.all(prices >= 0) + assert np.all(prices <= stock) + assert np.all(prices[(stock < 40_000) | (weights == 0)] == 0) + assert np.all(prices[prices > 0] >= 40_000) + assert np.any(prices > 0) + assert np.any((prices > 0) & (prices < stock)) + _assert_aggregate_targets( + prices, + weights, + sdlt_mask, + _targets(property_type, year), + tolerance=0.10, + value_tolerance=1e-8, + ) + + +@pytest.mark.unit +def test_property_types_have_independent_purchase_draws( + synthetic_households, monkeypatch +): + from policyengine_uk_data.datasets.imputations import property_purchases + + original = load_parameter("stochastic", "additional_property_purchases", 2024) + # Equal targets and price bands isolate the independence of event seeds. + assumptions = { + property_type: { + **original["additional_residential"], + "purchase_seed": original[property_type]["purchase_seed"], + "price_seed": original[property_type]["price_seed"], + } + for property_type, _, _ in PROPERTY_TYPES + } + monkeypatch.setattr(property_purchases, "load_parameter", lambda *args: assumptions) + stock, weights, sdlt_mask = synthetic_households + residential = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, "additional_residential" + ) + non_residential = impute_property_purchase_values( + stock, weights, sdlt_mask, 2024, "non_residential" + ) + assert np.any((residential > 0) != (non_residential > 0)) + + +@dataclass +class _HouseholdDataset: + household: pd.DataFrame + time_period: int = 2024 + + def copy(self): + return _HouseholdDataset(self.household.copy(), self.time_period) + + +@pytest.mark.unit +def test_purchase_inputs_do_not_depend_on_main_residence_purchase( + synthetic_households, +): + stock, weights, sdlt_mask = synthetic_households + original = _HouseholdDataset( + pd.DataFrame( + { + "region": np.where(sdlt_mask, "LONDON", "SCOTLAND"), + "household_weight": weights, + "other_residential_property_value": stock, + "non_residential_property_value": stock, + "property_purchased": False, + } + ) + ) + changed = original.copy() + changed.household["property_purchased"] = True + first = impute_property_purchases(original) + second = impute_property_purchases(changed) + for property_type, _, purchase_name in PROPERTY_TYPES: + assert purchase_name not in original.household + np.testing.assert_array_equal( + first.household[purchase_name], second.household[purchase_name] + ) + _assert_aggregate_targets( + first.household[purchase_name].to_numpy(), + weights, + sdlt_mask, + _targets(property_type, 2024), + tolerance=0.10, + value_tolerance=1e-8, + ) + assert not original.household.property_purchased.any() + assert second.household.property_purchased.all() + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +def test_insufficient_owner_population_fails(property_type): + with pytest.raises(ValueError, match="Insufficient weighted property owners"): + impute_property_purchase_values( + np.full(100, 1_000_000.0), + np.ones(100), + np.ones(100, dtype=bool), + 2024, + property_type, + ) + + +@pytest.mark.unit +@pytest.mark.parametrize("property_type", PROPERTY_TYPE_NAMES) +def test_stock_cap_prevents_infeasible_value_target( + synthetic_households, property_type +): + stock, weights, sdlt_mask = synthetic_households + with pytest.raises(ValueError, match="stock cannot support"): + impute_property_purchase_values( + np.where(stock > 0, 40_000.0, 0.0), + weights, + sdlt_mask, + 2024, + property_type, + ) + + +@pytest.mark.unit +@pytest.mark.parametrize( + "stock,weights", + [ + (np.array([np.nan]), np.ones(1)), + (np.ones(1), np.array([np.inf])), + (np.array([-1.0]), np.ones(1)), + (np.ones(1), np.array([-1.0])), + ], +) +def test_invalid_stock_or_weights_fail(stock, weights): + with pytest.raises(ValueError, match="finite and non-negative"): + impute_property_purchase_values( + stock, weights, np.ones(1, dtype=bool), 2024, "additional_residential" + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("property_type,stock_name,purchase_name", PROPERTY_TYPES) +@pytest.mark.parametrize("year", RELEASE_YEARS) +def test_saved_enhanced_frs_purchase_inputs( + enhanced_frs, baseline, property_type, stock_name, purchase_name, year +): + """A rebuilt dataset retains the input and targets after calibration.""" + input_columns = set(enhanced_frs.household.columns) + assert purchase_name in input_columns, ( + f"Enhanced FRS lacks {purchase_name}; rebuild with purchase imputation" + ) + prices = baseline.calculate(purchase_name, year).values + stock = baseline.calculate(stock_name, year).values + weights = baseline.calculate("household_weight", year).values + country = baseline.calculate("country", year, map_to="household").values + sdlt_mask = np.isin(country, ["ENGLAND", "NORTHERN_IRELAND"]) + # Store only aggregate booleans in assertions: pytest otherwise includes + # the household arrays in its rewritten assertion failure diagnostics. + finite = bool(np.all(np.isfinite(prices))) + non_negative = bool(np.all(prices >= 0)) + stock_capped = bool(np.all(prices <= stock + 0.01)) + owners_only = bool(np.all(prices[stock <= 0] == 0)) + assert finite, "Non-finite purchase inputs" + assert non_negative, "Negative purchase prices" + assert stock_capped, "Purchase prices exceed property stocks" + assert owners_only, "Non-owners purchase property" + _assert_aggregate_targets( + prices, weights, sdlt_mask, _targets(property_type, year), tolerance=0.20 + ) + + +@pytest.mark.integration +@pytest.mark.parametrize("year", RELEASE_YEARS) +def test_saved_enhanced_frs_higher_rates_sdlt(baseline, year): + """Gross HRAD SDLT is non-zero and near the allocated household benchmark. + + The 30% tolerance allows price-distribution and calibration differences, and + the annual model scale versus HMRC's fiscal-year rate changes and refunds. + """ + prices = baseline.calculate( + "additional_residential_property_purchased", year + ).values + weights = baseline.calculate("household_weight", year).values + country = baseline.calculate("country", year, map_to="household").values + sdlt_mask = np.isin(country, ["ENGLAND", "NORTHERN_IRELAND"]) + additional = baseline.tax_benefit_system.parameters( + f"{year}-01-01" + ).gov.hmrc.stamp_duty.residential.purchase.additional + liability = np.where(prices >= additional.min, additional.rate.calc(prices), 0.0) + revenue = float(np.dot(liability[sdlt_mask], weights[sdlt_mask])) + target = _targets("additional_residential", year)["revenue"] + assert revenue > 0, "Additional-dwelling purchases have no higher-rates SDLT" + assert revenue == pytest.approx(target, rel=0.30), ( + f"Weighted higher-rates SDLT £{revenue / 1e9:.3f}bn; " + f"household benchmark £{target / 1e9:.3f}bn; tolerance 30%" + ) diff --git a/scripts/validate_property_purchases.py b/scripts/validate_property_purchases.py new file mode 100644 index 000000000..8757edadc --- /dev/null +++ b/scripts/validate_property_purchases.py @@ -0,0 +1,181 @@ +"""Validate purchase imputation on a local enhanced FRS, printing aggregates only. + +Run from the repository root with the desired policyengine-uk on PYTHONPATH:: + + python scripts/validate_property_purchases.py /path/enhanced_frs_2023_24.h5 \ + --years 2023 2024 2025 + +Only four household columns are read. No dataset is downloaded or written. +Later years uprate stocks and weights with the model's dataset indices before +re-imputing purchases against that year's assumptions. This is a validation +of annual imputation, rather than a rebuilt or recalibrated dataset. +""" + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +# Make direct execution from scripts/ resolve this checkout. +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from policyengine_uk import parameters as model_parameters + +from policyengine_uk_data.datasets.imputations.property_purchases import ( + SDLT_REGIONS, + impute_property_purchase_values, +) +from policyengine_uk_data.parameters import load_parameter + +PROPERTY_STOCKS = { + "additional_residential": "other_residential_property_value", + "non_residential": "non_residential_property_value", +} +STOCK_INDEX = "gov.economic_assumptions.yoy_growth.obr.per_capita.gdp" +WEIGHT_INDEX = "gov.economic_assumptions.yoy_growth.ons.population" + + +def growth_factor(parameters, index, source_year, year): + """Use the same year-on-year factors as dataset projection.""" + return float( + np.prod( + [ + 1 + parameters.get_child(index)(str(y)) + for y in range(source_year + 1, year + 1) + ] + ) + ) + + +def validate(household, source_year, year, parameters): + """Return weighted E+NI aggregates and checks; never return record values.""" + if year < source_year: + raise ValueError("Validation years must not precede the dataset year") + stock_factor = growth_factor(parameters, STOCK_INDEX, source_year, year) + weight_factor = growth_factor(parameters, WEIGHT_INDEX, source_year, year) + weights = household.household_weight.to_numpy(dtype=float) * weight_factor + sdlt_mask = household.region.astype(str).isin(SDLT_REGIONS).to_numpy() + stamp_duty = parameters(str(year)).gov.hmrc.stamp_duty + rows = [] + for property_type, stock_name in PROPERTY_STOCKS.items(): + assumptions = load_parameter("stochastic", "additional_property_purchases", year)[ + property_type + ] + stock = household[stock_name].to_numpy(dtype=float) * stock_factor + prices = impute_property_purchase_values( + stock, weights, sdlt_mask, year, property_type + ) + repeat = impute_property_purchase_values( + stock, weights, sdlt_mask, year, property_type + ) + purchased = prices > 0 + minimum = assumptions["minimum_price"] + eligible = (stock >= minimum) & (weights > 0) + fitted_weights = weights * (purchased & sdlt_mask) + transactions = float(fitted_weights.sum()) + value = float(np.dot(prices, weights * sdlt_mask)) + checks = { + "reproducible": bool(np.array_equal(prices, repeat)), + "owners_only": bool(np.all(~purchased | eligible)), + "price_bounds": bool(np.all(prices <= stock)), + "minimum_price": bool(np.all(prices[purchased] >= minimum)), + "transactions_within_5_percent": bool( + abs(transactions / assumptions["transactions"] - 1) <= 0.05 + ), + "value_within_5_percent": bool( + abs(value / assumptions["value"] - 1) <= 0.05 + ), + } + if not all(checks.values()): + raise ValueError(f"Aggregate validation failed for {year} {property_type}") + if property_type == "additional_residential": + purchase = stamp_duty.residential.purchase.additional + liability = purchase.rate.calc(prices) * (prices >= purchase.min) + else: + liability = stamp_duty.non_residential.purchase.calc(prices) + bands = assumptions["price_bands"] + counts = np.array([band["transactions"] for band in bands], dtype=float) + means = np.array([band["value"] for band in bands]) / counts + raw_prices = np.random.default_rng(assumptions["price_seed"]).choice( + means, size=len(stock), p=counts / counts.sum() + ) + uncapped = purchased & (prices < stock) & (prices > minimum) + price_scale = ( + float(np.median(prices[uncapped] / raw_prices[uncapped])) + if uncapped.any() + else None + ) + rows.append( + { + "year": year, + "property_type": property_type, + "target_transactions": assumptions["transactions"], + "transactions": transactions, + "target_value": assumptions["value"], + "value": value, + "reference_gross_receipts": assumptions["revenue"], + "model_gross_sdlt": float(np.dot(liability, weights * sdlt_mask)), + "two_percentage_point_hrad_exposure": ( + 0.02 * value if property_type == "additional_residential" else None + ), + "eligible_weighted_owners": float(np.dot(eligible, weights * sdlt_mask)), + "eligible_mean_stock": float( + np.dot(stock, weights * (eligible & sdlt_mask)) + / np.dot(eligible, weights * sdlt_mask) + ), + "selected_stock_capacity": float(np.dot(stock, fitted_weights)), + "weighted_purchase_rate": float( + transactions / np.dot(eligible, weights * sdlt_mask) + ), + "price_scale": price_scale, + "weighted_stock_capped_share": float( + np.dot(purchased & (prices == stock), weights * sdlt_mask) + / transactions + ), + "stock_uprating_factor": stock_factor, + "weight_uprating_factor": weight_factor, + "checks": checks, + } + ) + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dataset", type=Path) + parser.add_argument("--years", type=int, nargs="+") + args = parser.parse_args() + source_year = int(pd.read_hdf(args.dataset, key="time_period").iloc[0]) + household = pd.read_hdf( + args.dataset, + key="household", + columns=["household_weight", "region", *PROPERTY_STOCKS.values()], + ) + rows = [ + row + for year in args.years or [source_year] + for row in validate(household, source_year, year, model_parameters) + ] + print( + json.dumps( + { + "source_year": source_year, + "scope": "England and Northern Ireland; weighted aggregates only", + "receipts_note": ( + "Model gross SDLT uses the policyengine-uk 1 January higher-rates " + "scale (the full tax, including standard rates). HMRC receipts " + "cover a financial year, policy changes and replacement-home " + "refund timing, so they are reference benchmarks, not fitted targets." + ), + "results": rows, + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() From 07f6b460f9db44c13bcce51c8c173174d3a38ec1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 20:30:14 -0400 Subject: [PATCH 6/6] Keep focused validation imports light and expose aggregate diagnostics Defer the uprating type-only dataset import and record price-band counts, stock capacity, price adjustment, caps and positive HRAD reform exposure in aggregate validation output. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_additional_property_purchased.py | 1 - policyengine_uk_data/utils/uprating.py | 11 +++++++++-- scripts/validate_property_purchases.py | 11 +++++++---- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/policyengine_uk_data/tests/test_additional_property_purchased.py b/policyengine_uk_data/tests/test_additional_property_purchased.py index f0c4df83b..c0894c95b 100644 --- a/policyengine_uk_data/tests/test_additional_property_purchased.py +++ b/policyengine_uk_data/tests/test_additional_property_purchased.py @@ -18,7 +18,6 @@ ) from policyengine_uk_data.parameters import load_parameter - PROPERTY_TYPES = ( ( "additional_residential", diff --git a/policyengine_uk_data/utils/uprating.py b/policyengine_uk_data/utils/uprating.py index 53c7e4015..1349778ec 100644 --- a/policyengine_uk_data/utils/uprating.py +++ b/policyengine_uk_data/utils/uprating.py @@ -1,6 +1,13 @@ -from policyengine_uk_data.storage import STORAGE_FOLDER +from __future__ import annotations + +from typing import TYPE_CHECKING + import pandas as pd -from policyengine_uk.data import UKSingleYearDataset + +from policyengine_uk_data.storage import STORAGE_FOLDER + +if TYPE_CHECKING: + from policyengine_uk.data import UKSingleYearDataset START_YEAR = 2020 END_YEAR = 2034 diff --git a/scripts/validate_property_purchases.py b/scripts/validate_property_purchases.py index 8757edadc..6e715936f 100644 --- a/scripts/validate_property_purchases.py +++ b/scripts/validate_property_purchases.py @@ -61,9 +61,9 @@ def validate(household, source_year, year, parameters): stamp_duty = parameters(str(year)).gov.hmrc.stamp_duty rows = [] for property_type, stock_name in PROPERTY_STOCKS.items(): - assumptions = load_parameter("stochastic", "additional_property_purchases", year)[ - property_type - ] + assumptions = load_parameter( + "stochastic", "additional_property_purchases", year + )[property_type] stock = household[stock_name].to_numpy(dtype=float) * stock_factor prices = impute_property_purchase_values( stock, weights, sdlt_mask, year, property_type @@ -121,7 +121,9 @@ def validate(household, source_year, year, parameters): "two_percentage_point_hrad_exposure": ( 0.02 * value if property_type == "additional_residential" else None ), - "eligible_weighted_owners": float(np.dot(eligible, weights * sdlt_mask)), + "eligible_weighted_owners": float( + np.dot(eligible, weights * sdlt_mask) + ), "eligible_mean_stock": float( np.dot(stock, weights * (eligible & sdlt_mask)) / np.dot(eligible, weights * sdlt_mask) @@ -130,6 +132,7 @@ def validate(household, source_year, year, parameters): "weighted_purchase_rate": float( transactions / np.dot(eligible, weights * sdlt_mask) ), + "price_band_count": len(bands), "price_scale": price_scale, "weighted_stock_capped_share": float( np.dot(purchased & (prices == stock), weights * sdlt_mask)