Impute additional-dwelling and non-residential purchases, calibrated to HMRC's HRAD statistics

changelog.d/556.added.md

@@ -0,0 +1 @@
+Impute seeded additional-dwelling and non-residential purchase prices from household property stocks, fitting explicit HMRC household transaction and value assumptions with a hard stock cap.

policyengine_uk_data/datasets/create_datasets.py

@@ -111,6 +111,7 @@ def main():
             "Impute student loan plan",
             "Assign Pension Credit take-up",
             "Clone and assign OA geography",
+            "Impute property purchases",
             "Calibrate constituency weights",
             "Calibrate local authority weights",
             "Export long local geography weights",
@@ -163,6 +164,7 @@ def main():
                 impute_salary_sacrifice,
                 impute_student_loan_plan,
                 uprate_property_by_region,
+                impute_property_purchases,
             )
 
             # Apply imputations with progress tracking
@@ -247,6 +249,12 @@ def main():
                 )
                 update_dataset(materialize_calibration_step, "completed")
 
+            # Stocks and final household geography now exist at the calibration
+            # year. Supply independent purchase inputs before fitting SDLT.
+            update_dataset("Impute property purchases", "processing")
+            frs = impute_property_purchases(frs)
+            update_dataset("Impute property purchases", "completed")
+
             # Calibrate constituency weights with nested progress
 
             update_dataset("Calibrate constituency weights", "processing")
@@ -417,7 +425,7 @@ def main():
                 "tiny_base_dataset": frs_release.tiny_base_dataset_file,
                 "tiny_enhanced_dataset": frs_release.tiny_enhanced_dataset_file,
                 "long_geography_weights": "local_geography_weights.csv.gz",
-                "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan",
+                "imputations_applied": "consumption, wealth, VAT, services, income, capital_gains, cgt_band_donors, salary_sacrifice, student_loan_plan, property_purchases",
                 "calibration": "national, LA and  constituency targets",
                 "pension_credit_take_up": pension_credit_takeup,
             },

policyengine_uk_data/datasets/imputations/__init__.py

@@ -8,3 +8,4 @@
 from .salary_sacrifice import impute_salary_sacrifice
 from .student_loans import impute_student_loan_plan
 from .regional_property_uprating import uprate_property_by_region
+from .property_purchases import impute_property_purchases

policyengine_uk_data/datasets/imputations/property_purchases.py

@@ -0,0 +1,147 @@
+"""Impute one annual purchase from each kind of household property stock.
+
+HMRC targets cover England and Northern Ireland. Independent seeded event
+draws are calibrated there; the resulting rates also apply in Scotland and
+Wales. Prices are separate seeded draws, never the household's entire stock
+by default. Assumptions and sources live in stochastic parameters.
+"""
+
+import numpy as np
+
+from policyengine_uk_data.parameters import load_parameter
+
+SDLT_REGIONS = (
+    "NORTH_EAST",
+    "NORTH_WEST",
+    "YORKSHIRE",
+    "EAST_MIDLANDS",
+    "WEST_MIDLANDS",
+    "EAST_OF_ENGLAND",
+    "LONDON",
+    "SOUTH_EAST",
+    "SOUTH_WEST",
+    "NORTHERN_IRELAND",
+)
+
+
+def impute_property_purchase_values(
+    stock: np.ndarray,
+    weights: np.ndarray,
+    sdlt_mask: np.ndarray,
+    year: int,
+    property_type: str,
+) -> np.ndarray:
+    """Return seeded purchase prices, fitting weighted events and capped value.
+
+    Purchase hazard is proportional to stock, an explicit portfolio-turnover
+    assumption. The closest weighted count determines the event threshold.
+    The value target is fitted by multiplying sampled band-average prices,
+    with the stock cap applied throughout. Zero-weight donors never purchase.
+    """
+    stock = np.asarray(stock, dtype=float)
+    weights = np.asarray(weights, dtype=float)
+    sdlt_mask = np.asarray(sdlt_mask, dtype=bool)
+    if (
+        stock.ndim != 1
+        or stock.shape != weights.shape
+        or stock.shape != sdlt_mask.shape
+    ):
+        raise ValueError(
+            "Property stocks, weights and SDLT mask must be aligned vectors"
+        )
+    if (
+        not np.all(np.isfinite(stock))
+        or not np.all(np.isfinite(weights))
+        or np.any(stock < 0)
+        or np.any(weights < 0)
+    ):
+        raise ValueError("Property stocks and weights must be finite and non-negative")
+
+    assumptions = load_parameter("stochastic", "additional_property_purchases", year)[
+        property_type
+    ]
+    target_count = assumptions["transactions"]
+    target_value = assumptions["value"]
+    minimum_price = assumptions["minimum_price"]
+    owners = (stock >= minimum_price) & (weights > 0)
+    eligible = np.flatnonzero(owners & sdlt_mask)
+    if weights[eligible].sum() < target_count:
+        raise ValueError("Insufficient weighted property owners for purchase target")
+
+    reference_stock = np.average(stock[eligible], weights=weights[eligible])
+    hazard = (np.maximum(stock, minimum_price) / reference_stock) ** assumptions[
+        "purchase_stock_exponent"
+    ]
+    uniform = np.random.default_rng(assumptions["purchase_seed"]).random(len(stock))
+    events = -np.log1p(-uniform) / hazard
+    ranked = eligible[np.argsort(events[eligible], kind="stable")]
+    counts = np.concatenate(([0.0], np.cumsum(weights[ranked])))
+    n_purchases = int(np.argmin(np.abs(counts - target_count)))
+    if n_purchases == len(ranked):
+        threshold = np.inf
+    elif n_purchases == 0:
+        threshold = 0.0
+    else:
+        threshold = (events[ranked[n_purchases - 1]] + events[ranked[n_purchases]]) / 2
+    purchased = owners & (events < threshold)
+    fitted_weights = weights * (purchased & sdlt_mask)
+    capacity = np.dot(stock, fitted_weights)
+    if capacity < target_value:
+        raise ValueError(
+            "Selected property stock cannot support the purchase value target"
+        )
+    if minimum_price * fitted_weights.sum() > target_value:
+        raise ValueError("Purchase value target is below the minimum transaction price")
+
+    bands = assumptions["price_bands"]
+    band_counts = np.array([band["transactions"] for band in bands], dtype=float)
+    band_prices = np.array([band["value"] for band in bands]) / band_counts
+    price_rng = np.random.default_rng(assumptions["price_seed"])
+    prices = price_rng.choice(
+        band_prices, size=len(stock), p=band_counts / band_counts.sum()
+    )
+
+    def capped_prices(scale):
+        return np.minimum(stock, np.maximum(minimum_price, prices * scale))
+
+    def weighted_value(scale):
+        return np.dot(capped_prices(scale), fitted_weights)
+
+    lower, upper = 0.0, 1.0
+    while weighted_value(upper) < target_value:
+        upper *= 2
+    for _ in range(80):
+        midpoint = (lower + upper) / 2
+        if weighted_value(midpoint) < target_value:
+            lower = midpoint
+        else:
+            upper = midpoint
+    return np.where(purchased, capped_prices(upper), 0.0)
+
+
+def impute_property_purchases(dataset, year=None):
+    """Add annual purchase inputs after wealth imputation, before calibration."""
+    dataset = dataset.copy()
+    year = int(dataset.time_period) if year is None else year
+    household = dataset.household
+    sdlt_mask = household["region"].astype(str).isin(SDLT_REGIONS).values
+    for property_type, stock_name, purchase_name in (
+        (
+            "additional_residential",
+            "other_residential_property_value",
+            "additional_residential_property_purchased",
+        ),
+        (
+            "non_residential",
+            "non_residential_property_value",
+            "non_residential_property_purchased",
+        ),
+    ):
+        household[purchase_name] = impute_property_purchase_values(
+            household[stock_name].values,
+            household["household_weight"].values,
+            sdlt_mask,
+            year,
+            property_type,
+        )
+    return dataset

policyengine_uk_data/parameters/stochastic/additional_property_purchases.yaml

@@ -0,0 +1,104 @@
+description: Household additional-dwelling and non-residential purchase assumptions
+metadata:
+  label: Additional property purchases
+  reference:
+    - title: HMRC Annual Stamp Tax Statistics 2024 to 2025, Tables 6b, 8a and 8b
+      href: https://www.gov.uk/government/statistics/uk-stamp-tax-statistics
+    - title: HMRC Annual Stamp Tax Statistics 2024 to 2025, corrected tables
+      href: https://assets.publishing.service.gov.uk/media/69aef72fc78869bf8eb8a589/Annual_Stamps_2024_25_correction.ods
+
+# HMRC SDLT tables cover England and Northern Ireland (E+NI), transactions
+# of at least GBP 40,000, and financial years ending in the following March.
+# Transaction counts are rounded to 100; values and receipts to GBP 5 million.
+# All values below are pounds. Imputation fits E+NI weighted counts and values.
+# Mean annual purchase rate is target transactions / eligible E+NI weighted
+# owners of stock of at least GBP 40,000. Stock-proportional purchase hazard
+# (exponent 1) assumes larger portfolios are more likely to trade each year;
+# it is a modelling choice, not an observed household transaction propensity.
+# Independent seeded exponential event scores divided by relative stock
+# select the closest attainable weighted count. The same hazard and threshold
+# apply elsewhere in the UK, an extrapolation without devolved targets.
+# Uniform owner selection on the local enhanced FRS 2023-24 selected too little
+# stock to support the assumed HRAD value. Stock weighting addresses that
+# feasibility issue without tuning seeds to a particular dataset.
+# Prices start from a seeded draw among band means (band value / transactions).
+# A common scale then fits the weighted purchase value with price equal to
+# min(stock, max(40,000, scale * draw)), so prices stay within HMRC's
+# transaction coverage and never exceed the owner's stock.
+# This is a capped, calibrated band-mean method, not an exact draw from the
+# price-band distribution conditional on stock. Rounded band counts are
+# normalised by their sum rather than by the headline count.
+#
+# Household HRAD is not separately published. The modelling choice is neutral
+# allocation using individuals' shares of all residential buyers in Table 8a:
+# transactions = 211,700 * 925,100 / (925,100 + 124,500) = 186,588.8624;
+# value = 77,970,000,000 * 335,515 / (335,515 + 44,690) = 68,805,261,766.6785.
+# Revenue is a reference benchmark, not fitted by imputation:
+# 5,435,000,000 * 9,145 / (9,145 + 1,230) = 4,790,657,831.3253.
+# This allocation is an assumption, not an observed household HRAD series.
+# Table 6b reports all-buyer gross HRAD SDLT before replacement-home refunds;
+# its surcharge-only receipts are GBP 2.790bn. The independent household lower
+# bounds implied by Tables 6b/8a are 87,200 transactions, GBP 33.280bn value,
+# and GBP 4.205bn gross receipts. They are bounds, not the selected targets.
+# For review, 2024-25 household HRAD targets and price bands are held flat across
+# survey/calibration years: there is no annual official household HRAD series
+# and this assumption is not a forecast. Non-residential targets use the
+# directly observed Table 8b individuals rows for 2023-24 and 2024-25, replacing
+# the older household rows in policyengine-uk's stamp_duty/statistics.yaml.
+# Their price bands also use individuals only. The 2024-25 band counts sum to
+# 25,100 rather than the 25,000 headline because of rounding; probabilities
+# use that band sum. Stock caps and the fitted scale modify the band distribution.
+values:
+  2013-01-01:
+    additional_residential: &additional_residential
+      transactions: 186588.8624237805
+      value: 68805261766.6785
+      revenue: 4790657831.325301
+      minimum_price: 40000
+      purchase_stock_exponent: 1
+      purchase_seed: 55601
+      price_seed: 55602
+      # Table 6b 2024-25: last band combines all five published bands over GBP 2m.
+      # Band sums differ slightly from headline totals because of rounding.
+      price_bands:
+        - {transactions: 113500, value: 16635000000} # Up to GBP 250,000
+        - {transactions: 60700, value: 21510000000} # GBP 250,001-500,000
+        - {transactions: 27500, value: 18770000000} # GBP 500,001-1,000,000
+        - {transactions: 5300, value: 6545000000} # GBP 1,000,001-1,500,000
+        - {transactions: 2000, value: 3515000000} # GBP 1,500,001-2,000,000
+        - {transactions: 2600, value: 10990000000} # Over GBP 2,000,000
+    non_residential:
+      # 2023-24 Table 8b individuals: 24,800 / GBP 6.955bn / GBP 230m.
+      # Table 8b individuals' price-band means, before stock-capped scaling.
+      transactions: 24800
+      value: 6955000000
+      revenue: 230000000
+      minimum_price: 40000
+      purchase_stock_exponent: 1
+      purchase_seed: 55603
+      price_seed: 55604
+      price_bands:
+        - {transactions: 18800, value: 1545000000} # Up to GBP 250,000
+        - {transactions: 3300, value: 1150000000} # GBP 250,001-500,000
+        - {transactions: 1500, value: 1085000000} # GBP 500,001-1,000,000
+        - {transactions: 500, value: 640000000} # GBP 1,000,001-1,500,000
+        - {transactions: 200, value: 415000000} # GBP 1,500,001-2,000,000
+        - {transactions: 500, value: 2120000000} # Over GBP 2,000,000
+  2024-01-01:
+    additional_residential: *additional_residential
+    non_residential:
+      # 2024-25 Table 8b individuals: 25,000 / GBP 7.640bn / GBP 245m.
+      transactions: 25000
+      value: 7640000000
+      revenue: 245000000
+      minimum_price: 40000
+      purchase_stock_exponent: 1
+      purchase_seed: 55603
+      price_seed: 55604
+      price_bands:
+        - {transactions: 18800, value: 1560000000} # Up to GBP 250,000
+        - {transactions: 3400, value: 1210000000} # GBP 250,001-500,000
+        - {transactions: 1600, value: 1100000000} # GBP 500,001-1,000,000
+        - {transactions: 500, value: 650000000} # GBP 1,000,001-1,500,000
+        - {transactions: 300, value: 445000000} # GBP 1,500,001-2,000,000
+        - {transactions: 500, value: 2675000000} # Over GBP 2,000,000

policyengine_uk_data/storage/uprating_factors.csv

@@ -1,4 +1,5 @@
 Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034
+additional_residential_property_purchased,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 afcs_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 alcohol_and_tobacco_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38
 benunit_rent,1.0,1.0,1.0,1.11,1.184,1.223,1.275,1.312,1.351,1.392,1.392,1.392,1.392,1.392,1.392
@@ -49,6 +50,7 @@ miscellaneous_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384
 mortgage_capital_repayment,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 mortgage_interest_repayment,1.0,1.0,1.262,1.874,2.288,2.599,2.927,3.167,3.3,3.455,3.455,3.455,3.455,3.455,3.455
 net_financial_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
+non_residential_property_purchased,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 non_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 other_investment_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384
 other_residential_property_value,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384

policyengine_uk_data/storage/uprating_growth_factors.csv

@@ -1,4 +1,5 @@
 Variable,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034
+additional_residential_property_purchased,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 afcs_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 alcohol_and_tobacco_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0
 benunit_rent,0,0.0,0.0,0.11,0.067,0.033,0.043,0.029,0.03,0.03,0.0,0.0,0.0,0.0,0.0
@@ -49,6 +50,7 @@ miscellaneous_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.
 mortgage_capital_repayment,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 mortgage_interest_repayment,0,0.0,0.262,0.485,0.221,0.136,0.126,0.082,0.042,0.047,0.0,0.0,0.0,0.0,0.0
 net_financial_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
+non_residential_property_purchased,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 non_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 other_investment_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0
 other_residential_property_value,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0