From 2ad43a68f7bd6f8ef4e9e954678c8a666572f9d4 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 30 Sep 2026 22:22:44 -0400 Subject: [PATCH 1/4] Split pensions, shares, trusts, ISAs and secured debt out of WAS wealth corporate_wealth was built as (WAS total private pension wealth less current-employment DB) + employee shares + UK shares + investment ISAs + unit/investment trusts, so 84.7% of its WAS round 8 mass (GBP 3,659bn of 4,318bn) was pension wealth, which every means test disregards (UC Regs 2013 Sch 10 para 10 and the legacy equivalents). policyengine-uk counts corporate_wealth as capital, so imputed pensions failed claimants on the GBP 16,000 limit. The imputation now draws private_pension_wealth in the slot the old column held, then directly_held_shares (UK + employee shares), unit_and_investment_ trusts and stocks_and_shares_isa, and builds corporate_wealth as their exact sum. These are the parts DWP values differently: quoted shares less 10% for costs of sale (ADM H1665), unit trusts with none (H1674), ISAs at withdrawal value (H1656). cash_isa is exported too. other_residential_property_value took only WAS second homes (DVHseValR8, GBP 254bn) and left buy-to-let (DVBltValR8, GBP 573bn) out of every capital input; it now holds both. The debt secured on each non-main asset class is imputed after its asset for the policyengine-uk inputs that value capital net of encumbrances (reg 49(1)(b)): second homes + buy-to-let, other buildings and UK land, kept only where the household holds the asset. The model metadata records how each derived target is built, so a cached model retrains when a definition changes. Uprating rows for the new columns copy corporate_wealth's per-capita GDP index. Co-Authored-By: Claude Opus 5.5 --- changelog.d/was-wealth-components.added.md | 1 + changelog.d/was-wealth-split.fixed.md | 1 + docs/imputations.md | 39 ++-- .../datasets/imputations/wealth.py | 107 ++++++++-- .../storage/uprating_factors.csv | 8 + .../storage/uprating_growth_factors.csv | 8 + .../tests/test_student_loan_balance.py | 11 +- .../tests/test_wealth_split.py | 185 ++++++++++++++++++ pyproject.toml | 1 + uv.lock | 73 ++++++- 10 files changed, 397 insertions(+), 37 deletions(-) create mode 100644 changelog.d/was-wealth-components.added.md create mode 100644 changelog.d/was-wealth-split.fixed.md create mode 100644 policyengine_uk_data/tests/test_wealth_split.py diff --git a/changelog.d/was-wealth-components.added.md b/changelog.d/was-wealth-components.added.md new file mode 100644 index 000000000..598d6c8fe --- /dev/null +++ b/changelog.d/was-wealth-components.added.md @@ -0,0 +1 @@ +Export `private_pension_wealth`, `directly_held_shares`, `unit_and_investment_trusts`, `stocks_and_shares_isa` and `cash_isa` from the Wealth and Assets Survey, and impute debt secured on non-main property and land (`other_residential_property_secured_debt`, `non_residential_property_secured_debt`, `owned_land_secured_debt`). diff --git a/changelog.d/was-wealth-split.fixed.md b/changelog.d/was-wealth-split.fixed.md new file mode 100644 index 000000000..83bc292bb --- /dev/null +++ b/changelog.d/was-wealth-split.fixed.md @@ -0,0 +1 @@ +Split private pension wealth out of `corporate_wealth`, which now holds only shares, unit and investment trusts and stocks and shares ISAs, and add buy-to-let property to `other_residential_property_value`. diff --git a/docs/imputations.md b/docs/imputations.md index 9e6964ed3..f09374d11 100644 --- a/docs/imputations.md +++ b/docs/imputations.md @@ -19,9 +19,9 @@ The imputations are applied in this order (dependencies noted): ## Wealth Imputation -**Source:** Wealth and Assets Survey (WAS) Round 7 (2018-2020) +**Source:** Wealth and Assets Survey (WAS) Round 8 (April 2020 to March 2022), household file -Imputes household wealth components using demographic and income predictors. +Imputes household wealth components using demographic and income predictors. The targets are imputed as a chain: each is fitted on the predictors and every earlier target. ### Predictors | Variable | Description | @@ -39,18 +39,29 @@ Imputes household wealth components using demographic and income predictors. | `region` | UK region | ### Outputs -| Variable | Description | -|----------|-------------| -| `owned_land` | Value of owned land | -| `property_wealth` | Total property wealth | -| `corporate_wealth` | Shares, pensions, investment ISAs | -| `gross_financial_wealth` | Total financial assets | -| `net_financial_wealth` | Financial assets minus liabilities | -| `main_residence_value` | Value of main home | -| `other_residential_property_value` | Value of other properties | -| `non_residential_property_value` | Value of non-residential property | -| `savings` | Savings account balances | -| `num_vehicles` | Number of vehicles owned | +| Variable | WAS round 8 source | Description | +|----------|--------------------|-------------| +| `owned_land` | `DVLUKValR8_sum` | Value of land in the UK | +| `property_wealth` | `DVPropertyR8` | Gross value of all property, main residence included | +| `private_pension_wealth` | `totalpenr8_aggr` − `dvvaldbt_scaper8_aggr` | Private pension wealth other than current-employment defined benefit rights: defined contribution pots, retained rights, pensions in payment | +| `directly_held_shares` | `DVFShUKVR8_aggr` + `DVFESHARESR8_aggr` | UK shares and employee shares and options, held outside ISAs and pooled funds | +| `unit_and_investment_trusts` | `DVFCollVR8_aggr` | Unit trusts and investment trusts (one survey question) | +| `stocks_and_shares_isa` | `DVIISAVR8_aggr` | Stocks and shares (investment) ISAs | +| `corporate_wealth` | derived | `directly_held_shares` + `unit_and_investment_trusts` + `stocks_and_shares_isa`, summed after imputation | +| `gross_financial_wealth` | `HFINWR8_SUM` | Total financial assets | +| `net_financial_wealth` | `HFINWNTR8_Sum` | Financial assets minus liabilities | +| `main_residence_value` | `DVHValueR8` | Value of main home | +| `other_residential_property_value` | `DVHseValR8_sum` + `DVBltValR8_sum` | Second homes and buy-to-let property | +| `non_residential_property_value` | `DVBlDValR8_sum` | Other buildings, such as shops and garages | +| `savings` | `DVSaValR8_aggr` | Savings account balances (cash ISAs excluded) | +| `num_vehicles` | `vcarnr8` | Number of vehicles owned | +| `student_loan_balance` | `Tot_LosR8_aggr` − `Tot_los_exc_SLCR8_aggr` | Student Loans Company balance, allocated to people | +| `cash_isa` | `DVCISAVR8_aggr` | Cash ISAs | +| `other_residential_property_secured_debt` | `DVHseDebtR8_sum` + `DVBLtDebtR8_sum` | Mortgages and loans secured on second homes and buy-to-let property | +| `non_residential_property_secured_debt` | `DVBldDebtR8_sum` | Mortgages and loans secured on other buildings | +| `owned_land_secured_debt` | `DVLUKDebtR8_sum` | Loans secured on UK land | + +`corporate_wealth` is the exact sum of its three components on every household; private pension wealth is not part of it, because pension rights are disregarded capital in every means test. A secured debt is zero wherever the household holds none of the asset it is secured on. Land and property overseas (`DVLOSValR8_sum`), other real estate (`DVOPrValR8_sum`) and overseas shares (`DVFShOSVR8_aggr`) are not mapped to any output. --- diff --git a/policyengine_uk_data/datasets/imputations/wealth.py b/policyengine_uk_data/datasets/imputations/wealth.py index 36c5fd4dc..4134d2b47 100644 --- a/policyengine_uk_data/datasets/imputations/wealth.py +++ b/policyengine_uk_data/datasets/imputations/wealth.py @@ -45,10 +45,17 @@ "region", ] +# Chain order matters: each target is fitted on the predictors plus every +# earlier target. Private pension wealth takes the slot the old pension-laden +# corporate_wealth held, the share-like components follow it, and each +# secured debt comes after the asset it is secured on. IMPUTE_VARIABLES = [ "owned_land", "property_wealth", - "corporate_wealth", + "private_pension_wealth", + "directly_held_shares", + "unit_and_investment_trusts", + "stocks_and_shares_isa", "gross_financial_wealth", "net_financial_wealth", "main_residence_value", @@ -57,17 +64,58 @@ "savings", "num_vehicles", "student_loan_balance", + "cash_isa", + "other_residential_property_secured_debt", + "non_residential_property_secured_debt", + "owned_land_secured_debt", ] +# corporate_wealth is not imputed: it is the sum of its imputed components, +# so the identity holds on every household. +CORPORATE_WEALTH_COMPONENTS = ( + "directly_held_shares", + "unit_and_investment_trusts", + "stocks_and_shares_isa", +) + +# Debt secured on each capital asset (UC Regs 2013 reg. 49(1)(b) and its +# legacy equivalents deduct it from that asset's value only). +SECURED_DEBT_ASSETS = { + "other_residential_property_secured_debt": "other_residential_property_value", + "non_residential_property_secured_debt": "non_residential_property_value", + "owned_land_secured_debt": "owned_land", +} + +# WAS round 8 sources of the targets built from more than one column. Part of +# the model metadata, so a change of definition retrains a cached model. +DERIVED_TARGETS = { + "private_pension_wealth": ("totalpenr8_aggr", "-dvvaldbt_scaper8_aggr"), + "directly_held_shares": ("DVFShUKVR8_aggr", "DVFESHARESR8_aggr"), + "other_residential_property_value": ("DVHseValR8_sum", "DVBltValR8_sum"), + "other_residential_property_secured_debt": ( + "DVHseDebtR8_sum", + "DVBLtDebtR8_sum", + ), + "student_loan_balance": ("Tot_LosR8_aggr", "-Tot_los_exc_SLCR8_aggr"), +} + WAS_RENAMES = { "R8xshhwgt": "household_weight", # Components for estimating land holdings. "DVLUKValR8_sum": "owned_land", # In the UK. + "DVLUKDebtR8_sum": "owned_land_secured_debt", "DVPropertyR8": "property_wealth", + # UK shares (listed or not) and employee shares and options, held outside + # ISAs and pooled funds. "DVFESHARESR8_aggr": "emp_shares_options", "DVFShUKVR8_aggr": "uk_shares", - "DVIISAVR8_aggr": "investment_isas", - "DVFCollVR8_aggr": "unit_investment_trusts", + # Investment ISAs: the survey's stocks and shares ISA question. + "DVIISAVR8_aggr": "stocks_and_shares_isa", + "DVCISAVR8_aggr": "cash_isa", + # Unit trusts and investment trusts: one survey question. + "DVFCollVR8_aggr": "unit_and_investment_trusts", + # Total private pension wealth and its current-employment defined benefit + # part, both valued on the SCAPE basis. "totalpenr8_aggr": "pensions", "dvvaldbt_scaper8_aggr": "db_pensions", # Predictors for fusing to FRS. @@ -88,12 +136,17 @@ # Other columns for reference. "DVLOSValR8_sum": "non_uk_land", "HFINWNTR8_Sum": "net_financial_wealth", - "DVLUKDebtR8_sum": "uk_land_debt", "HFINWR8_SUM": "gross_financial_wealth", "TotalWlthR8": "wealth", "DVhvalueR8": "main_residence_value", - "DVHseValR8_sum": "other_residential_property_value", + # Gross values of property other than the main residence, and the + # mortgages and loans secured on each class. + "DVHseValR8_sum": "second_homes_value", + "DVHseDebtR8_sum": "second_homes_debt", + "DVBltValR8_sum": "buy_to_let_value", + "DVBLtDebtR8_sum": "buy_to_let_debt", "DVBlDValR8_sum": "non_residential_property_value", + "DVBldDebtR8_sum": "non_residential_property_secured_debt", "DVTotinc_bhcR8": "household_net_income", "DVSaValR8_aggr": "savings", "vcarnr8": "num_vehicles", @@ -142,16 +195,18 @@ def generate_was_table(was: pd.DataFrame): was["is_renting"] = was["is_renter"] == 1 - was["non_db_pensions"] = was.pensions - was.db_pensions - was["corporate_wealth"] = was[ - [ - "non_db_pensions", - "emp_shares_options", - "uk_shares", - "investment_isas", - "unit_investment_trusts", - ] - ].sum(axis=1) + # Private pension wealth other than current-employment defined benefit + # rights: pension rights are disregarded capital in every means test, so it + # is kept out of corporate_wealth. + was["private_pension_wealth"] = was.pensions - was.db_pensions + was["directly_held_shares"] = was.uk_shares + was.emp_shares_options + was["corporate_wealth"] = was[list(CORPORATE_WEALTH_COMPONENTS)].sum(axis=1) + was["other_residential_property_value"] = ( + was.second_homes_value + was.buy_to_let_value + ) + was["other_residential_property_secured_debt"] = ( + was.second_homes_debt + was.buy_to_let_debt + ) was["student_loan_balance"] = was["total_loans"] - was["total_loans_exc_slc"] was["region"] = was["region"].map(REGIONS) return was @@ -162,6 +217,7 @@ def generate_was_table(was: pd.DataFrame): "was_household_tab_filename": CURRENT_WAS_RELEASE.household_tab_filename, "predictor_variables": tuple(PREDICTOR_VARIABLES), "impute_variables": tuple(IMPUTE_VARIABLES), + "derived_targets": tuple(DERIVED_TARGETS.items()), } @@ -269,6 +325,25 @@ def _allocate_student_loan_balance_to_people( return balances +def derive_wealth_outputs(output_df: pd.DataFrame) -> pd.DataFrame: + """ + Add the outputs derived from imputed targets. + + The share-like components are clipped at zero and summed into + corporate_wealth, so the identity holds on every household. A secured debt + is kept only where the household holds the asset it is secured on. + """ + output_df = output_df.copy() + for column in (*CORPORATE_WEALTH_COMPONENTS, *SECURED_DEBT_ASSETS): + output_df[column] = output_df[column].clip(lower=0) + output_df["corporate_wealth"] = output_df[list(CORPORATE_WEALTH_COMPONENTS)].sum( + axis=1 + ) + for debt, asset in SECURED_DEBT_ASSETS.items(): + output_df[debt] = output_df[debt].where(output_df[asset] > 0, 0.0) + return output_df + + def save_imputation_models(): """ Train and save wealth imputation model. @@ -338,7 +413,7 @@ def impute_wealth(dataset: UKSingleYearDataset) -> UKSingleYearDataset: input_df["region"] = input_df["region"].replace( "NORTHERN_IRELAND", "WALES" ) # WAS doesn't sample NI -> put NI households in Wales (closest aggregate) - output_df = model.predict(input_df) + output_df = derive_wealth_outputs(model.predict(input_df)) for column in output_df.columns: if column == "student_loan_balance": diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index dacd591bd..781c8f25a 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -6,6 +6,7 @@ bsp_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1 capital_gains,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 capital_gains_before_response,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 carers_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +cash_isa,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 child_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 child_tax_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 @@ -13,11 +14,15 @@ clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1 communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 diesel_spending,1.0,1.138,1.57,1.539,1.589,1.54,1.507,1.46,1.415,1.355,1.284,1.284,1.284,1.284,1.284 +directly_held_shares,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 employee_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 employer_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 +non_residential_property_secured_debt,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +other_residential_property_secured_debt,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +owned_land_secured_debt,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_contributions_via_salary_sacrifice,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 employment_income,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 employment_income_before_lsr,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 @@ -58,6 +63,7 @@ pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346 +private_pension_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 recreation_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 @@ -71,9 +77,11 @@ state_pension_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1 statutory_maternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 statutory_paternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 statutory_sick_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +stocks_and_shares_isa,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 student_loan_repayments,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 sublet_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 transport_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +unit_and_investment_trusts,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 universal_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 water_and_sewerage_charges,1.0,1.0,1.0,1.092,1.14,1.21,1.283,1.349,1.4,1.46,1.46,1.46,1.46,1.46,1.46 winter_fuel_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 122b4ed69..c4106cd0b 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -6,6 +6,7 @@ bsp_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0 capital_gains,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 capital_gains_before_response,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 carers_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +cash_isa,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 child_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 child_tax_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 @@ -13,11 +14,15 @@ clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0 communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 diesel_spending,0,0.138,0.379,-0.019,0.033,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 +directly_held_shares,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 employee_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 employer_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 +non_residential_property_secured_debt,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +other_residential_property_secured_debt,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +owned_land_secured_debt,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_contributions_via_salary_sacrifice,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 employment_income,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 employment_income_before_lsr,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 @@ -58,6 +63,7 @@ pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0, personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0 +private_pension_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 recreation_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 @@ -71,9 +77,11 @@ state_pension_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0 statutory_maternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 statutory_paternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 statutory_sick_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +stocks_and_shares_isa,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 student_loan_repayments,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 sublet_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 transport_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +unit_and_investment_trusts,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 universal_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 water_and_sewerage_charges,0,0.0,0.0,0.092,0.044,0.061,0.06,0.051,0.038,0.043,0.0,0.0,0.0,0.0,0.0 winter_fuel_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_student_loan_balance.py b/policyengine_uk_data/tests/test_student_loan_balance.py index ea95bdf61..160007772 100644 --- a/policyengine_uk_data/tests/test_student_loan_balance.py +++ b/policyengine_uk_data/tests/test_student_loan_balance.py @@ -170,13 +170,12 @@ class DummyModel: @staticmethod def predict(_input_df): - return pd.DataFrame( - { - "owned_land": [10.0, 20.0], - "student_loan_balance": [900.0, 300.0], - }, - index=[1, 2], + output = pd.DataFrame( + 0.0, index=[1, 2], columns=list(wealth.IMPUTE_VARIABLES) ) + output["owned_land"] = [10.0, 20.0] + output["student_loan_balance"] = [900.0, 300.0] + return output class DummyMicrosimulation: def __init__(self, dataset): diff --git a/policyengine_uk_data/tests/test_wealth_split.py b/policyengine_uk_data/tests/test_wealth_split.py new file mode 100644 index 000000000..1af7402fb --- /dev/null +++ b/policyengine_uk_data/tests/test_wealth_split.py @@ -0,0 +1,185 @@ +"""Invariants of the WAS wealth split: pensions out of corporate_wealth, the +share-like components, buy-to-let property and secured debts.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pandas as pd +from hypothesis import given, settings +from hypothesis import strategies as st + +_WEALTH_PATH = ( + Path(__file__).resolve().parents[1] / "datasets" / "imputations" / "wealth.py" +) +_WEALTH_SPEC = importlib.util.spec_from_file_location( + "wealth_split_module", _WEALTH_PATH +) +wealth = importlib.util.module_from_spec(_WEALTH_SPEC) +_WEALTH_SPEC.loader.exec_module(wealth) + +COMPONENTS = list(wealth.CORPORATE_WEALTH_COMPONENTS) +DEBTS = list(wealth.SECURED_DEBT_ASSETS) + +amount = st.floats(min_value=0, max_value=1e8, allow_nan=False) +signed_amount = st.floats(min_value=-1e7, max_value=1e8, allow_nan=False) + +WAS_SOURCE_COLUMNS = { + "pensions": "totalpenr8_aggr", + "db_pensions": "dvvaldbt_scaper8_aggr", + "uk_shares": "DVFShUKVR8_aggr", + "emp_shares": "DVFESHARESR8_aggr", + "isa": "DVIISAVR8_aggr", + "cash_isa": "DVCISAVR8_aggr", + "trusts": "DVFCollVR8_aggr", + "second_homes": "DVHseValR8_sum", + "second_homes_debt": "DVHseDebtR8_sum", + "btl": "DVBltValR8_sum", + "btl_debt": "DVBLtDebtR8_sum", + "buildings_debt": "DVBldDebtR8_sum", + "land_debt": "DVLUKDebtR8_sum", +} + + +def _was_row(**values): + row = {column: 0 for column in wealth.WAS_RENAMES} + row.update({"R8xshhwgt": 1, "GORR8": 11, "DVPriRntR8": 1}) + for key, value in values.items(): + row[WAS_SOURCE_COLUMNS[key]] = value + return row + + +@st.composite +def was_rows(draw): + pensions = draw(amount) + return _was_row( + pensions=pensions, + db_pensions=draw(st.floats(min_value=0, max_value=pensions)), + **{ + key: draw(amount) + for key in WAS_SOURCE_COLUMNS + if key not in ("pensions", "db_pensions") + }, + ) + + +def test_every_imputed_target_is_built_from_the_survey(): + was = wealth.generate_was_table(pd.DataFrame([_was_row()])) + missing = set(wealth.IMPUTE_VARIABLES) - set(was.columns) + assert not missing + + +def test_corporate_wealth_is_derived_not_imputed(): + assert "corporate_wealth" not in wealth.IMPUTE_VARIABLES + assert set(COMPONENTS) <= set(wealth.IMPUTE_VARIABLES) + assert "private_pension_wealth" in wealth.IMPUTE_VARIABLES + + +def test_each_secured_debt_is_imputed_after_its_asset(): + order = wealth.IMPUTE_VARIABLES + for debt, asset in wealth.SECURED_DEBT_ASSETS.items(): + assert order.index(asset) < order.index(debt) + + +def test_definitions_are_part_of_the_model_metadata(): + metadata = wealth.get_wealth_model_metadata() + assert dict(metadata["derived_targets"]) == wealth.DERIVED_TARGETS + + +def test_generate_was_table_maps_each_component(): + was = wealth.generate_was_table( + pd.DataFrame( + [ + _was_row( + pensions=500, + db_pensions=200, + uk_shares=10, + emp_shares=20, + isa=40, + cash_isa=7, + trusts=30, + second_homes=1_000, + second_homes_debt=100, + btl=2_000, + btl_debt=900, + buildings_debt=50, + land_debt=5, + ) + ] + ) + ).iloc[0] + assert was.private_pension_wealth == 300 + assert was.directly_held_shares == 30 + assert was.unit_and_investment_trusts == 30 + assert was.stocks_and_shares_isa == 40 + assert was.cash_isa == 7 + assert was.corporate_wealth == 100 + assert was.other_residential_property_value == 3_000 + assert was.other_residential_property_secured_debt == 1_000 + assert was.non_residential_property_secured_debt == 50 + assert was.owned_land_secured_debt == 5 + + +@settings(max_examples=200, deadline=None) +@given(was_rows()) +def test_split_conserves_the_old_corporate_wealth(row): + """Old corporate_wealth = new corporate_wealth + private_pension_wealth.""" + was = wealth.generate_was_table(pd.DataFrame([row])).iloc[0] + old_corporate_wealth = ( + row["totalpenr8_aggr"] + - row["dvvaldbt_scaper8_aggr"] + + row["DVFESHARESR8_aggr"] + + row["DVFShUKVR8_aggr"] + + row["DVIISAVR8_aggr"] + + row["DVFCollVR8_aggr"] + ) + assert np.isclose( + was.corporate_wealth + was.private_pension_wealth, + old_corporate_wealth, + rtol=1e-12, + atol=1e-6, + ) + assert np.isclose( + was.corporate_wealth, sum(was[c] for c in COMPONENTS), rtol=1e-12, atol=1e-6 + ) + assert was.private_pension_wealth >= 0 + assert ( + was.other_residential_property_value + == row["DVHseValR8_sum"] + row["DVBltValR8_sum"] + ) + + +@st.composite +def draws(draw): + n = draw(st.integers(min_value=1, max_value=20)) + columns = { + column: draw(st.lists(signed_amount, min_size=n, max_size=n)) + for column in (*COMPONENTS, *DEBTS, *wealth.SECURED_DEBT_ASSETS.values()) + } + # Exact zero assets must be reachable, since that is where debt is cleared. + for asset in wealth.SECURED_DEBT_ASSETS.values(): + zeros = draw(st.lists(st.booleans(), min_size=n, max_size=n)) + columns[asset] = [0.0 if z else v for z, v in zip(zeros, columns[asset])] + return pd.DataFrame(columns) + + +@settings(max_examples=300, deadline=None) +@given(draws()) +def test_derived_outputs_hold_their_identities(output): + derived = wealth.derive_wealth_outputs(output) + assert (derived[COMPONENTS] >= 0).all().all() + assert (derived[DEBTS] >= 0).all().all() + assert np.allclose( + derived.corporate_wealth, derived[COMPONENTS].sum(axis=1), rtol=0, atol=0 + ) + for debt, asset in wealth.SECURED_DEBT_ASSETS.items(): + no_asset = derived[asset] <= 0 + assert (derived.loc[no_asset, debt] == 0).all() + held = ~no_asset + assert np.array_equal( + derived.loc[held, debt], output.loc[held, debt].clip(lower=0) + ) + # Assets themselves are untouched, and deriving twice changes nothing. + for asset in wealth.SECURED_DEBT_ASSETS.values(): + assert derived[asset].equals(output[asset]) + pd.testing.assert_frame_equal(wealth.derive_wealth_outputs(derived), derived) diff --git a/pyproject.toml b/pyproject.toml index beff7f1a4..58de5b1d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ dev = [ "ruff>=0.9.0", "pytest", + "hypothesis", "torch", "l0-python>=0.4.0", "tables", diff --git a/uv.lock b/uv.lock index a3e9f44d9..c301c6c9f 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/35/f4/124858007ddf3c61e9b144107304c9152fa80b5b6c168da07d86fe583cc1/huggingface_hub-1.1.5-py3-none-any.whl", hash = "sha256:e88ecc129011f37b868586bbcfae6c56868cae80cd56a79d61575426a3aa0d7d", size = 516000, upload-time = "2025-11-20T15:49:30.926Z" }, ] +[[package]] +name = "hypothesis" +version = "6.168.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/b7/13118bbc45d6d8b9d04e2de779e2a4ff23145ea39efa692b994fb874ca72/hypothesis-6.168.3.tar.gz", hash = "sha256:a43388f9067678fef6e13bdff325b6cfa6961a590498bb37f7ff31589c83bc75", size = 511022, upload-time = "2026-09-28T05:20:58.499Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/cd/746a2fc1e5e5ba34f07ea6ee1ad07525a3dde5ff4836db3a7980af49a891/hypothesis-6.168.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:d20972ca134e652e9928ecd200966a8a2857adc2812c1d74b12a872e5cd503eb", size = 791536, upload-time = "2026-09-28T05:18:57.162Z" }, + { url = "https://files.pythonhosted.org/packages/c3/0f/7a158e377b69556c8e12c25c8fd811d0103e54c24a12dab1f3b9819f202e/hypothesis-6.168.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:eafbec09d3e87d13d8242411d1f5868b5e879f1bd6d95e233e9ff80c527bea1c", size = 787313, upload-time = "2026-09-28T05:20:18.122Z" }, + { url = "https://files.pythonhosted.org/packages/45/c6/7df9104ee359e8fcd77791781c47e70794964ca69673665e0de2a6590f4c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73c5627497968cc62d140e9fde1ea21e12cac7b64dc419fea8286cd34ff1ad4e", size = 1124036, upload-time = "2026-09-28T05:20:04.372Z" }, + { url = "https://files.pythonhosted.org/packages/92/0f/8a61715404a73b9a82cb1f976803428d603cdea976a9c30266c72ad6a7ce/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dc56e6dc6eeb0aeb08ac463f847279bde2336c4786faf7ee0af4b93cd031a7", size = 1147875, upload-time = "2026-09-28T05:19:15.495Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3f/2b16e95cc3b9a069afa23830a012aefe377ce4a457a227e5a5026eb827ff/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:753bb501f8560d2e321ed3b62596b4496c56f15668b0a0669231ccc3e6c4e80d", size = 1149489, upload-time = "2026-09-28T05:19:18.886Z" }, + { url = "https://files.pythonhosted.org/packages/f6/e7/d7cf6dc068bb02732b2a6e38a35a0dcb7f2137608198fc79d740f69d197c/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71ab606c472449cb872ef2a7acaec679ee4f651b7f0a477a04ebc85d8933ef8c", size = 1191992, upload-time = "2026-09-28T05:19:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/5f/26/f1e5b25dec15998e8375d722825ec3dae1075cbe7cb1f74221b2312a2e33/hypothesis-6.168.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26928956c54748e4dfa587333741ab750246123b93484955646ff316cca27eef", size = 1169911, upload-time = "2026-09-28T05:19:27.996Z" }, + { url = "https://files.pythonhosted.org/packages/03/36/901234a49147e6bf8a5b9e54b775706a223c2b52befea1106ea2aaf15d48/hypothesis-6.168.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:0bdfc53c041b61c854fa3761735736b991bc2bddafee45a361cfe4d43027c1ef", size = 1129406, upload-time = "2026-09-28T05:18:42.632Z" }, + { url = "https://files.pythonhosted.org/packages/55/c1/5fc9ff91ec9fba6bb527833386c66812524b5cc75fe7d4a4f9122ee1c420/hypothesis-6.168.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:650528e2b1e2a45e95c2df624d4b9364b8ac5a027ba84e1eea2bc5398dd01dcd", size = 1160399, upload-time = "2026-09-28T05:20:23.911Z" }, + { url = "https://files.pythonhosted.org/packages/9b/d2/49ad1ef5c55547c8abfd0fc571b3493806dda530fd5cb3d8263dbad86a72/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:209dc54cdb1b4d6d7020ad8d09e44c49756361b7183adacea4d6f5705085b595", size = 1299906, upload-time = "2026-09-28T05:18:58.417Z" }, + { url = "https://files.pythonhosted.org/packages/0d/5f/f98a26094c4f462c4215807bed0f9a6b5508b27a3c329bcba1bc8adc7ac7/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:af8ca98cd11dc7f9427bb90483abcd9688d4df2e8e63893a30a2423b027ebb11", size = 1425538, upload-time = "2026-09-28T05:20:49.074Z" }, + { url = "https://files.pythonhosted.org/packages/d0/b8/c5b4406e3a6591e51a42f85469351d1d824e0a54b84167aec9b2353759fe/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:f8122cfdd0bc0ba843063effa22ac310bdebdb3a1319bf1e63f14baa119c72ab", size = 1377084, upload-time = "2026-09-28T05:20:06.618Z" }, + { url = "https://files.pythonhosted.org/packages/e0/01/0d84a8ea469d024c15f602799c0b8410fbe03d99fdb0370449b33c3c916f/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4bedbb379eab34f792af7ee9a05aae04e9c08bbb52e0f90f5a2110d4fe4b2fbd", size = 1281162, upload-time = "2026-09-28T05:19:42.768Z" }, + { url = "https://files.pythonhosted.org/packages/22/b5/ea6435038de1a795f005cb603531e8ccc053febc50180f3ab55583c9c60e/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:5a953115b9f5c95133ab2d04efffeec96e5658c3207923dca7285f7db3e6bbef", size = 1300433, upload-time = "2026-09-28T05:18:55.634Z" }, + { url = "https://files.pythonhosted.org/packages/6c/78/fa6c77d9f64b6d592e69e582d83cbce7d7e56897faa4762885a2123b0166/hypothesis-6.168.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6173558e676ad25ed1e20507fa4024a0d816dd90f715f77006e5a28f19109026", size = 1336273, upload-time = "2026-09-28T05:19:08.273Z" }, + { url = "https://files.pythonhosted.org/packages/f1/56/4acd0d778bab818c9ea336fcba768bf43bc70db7f84cc373579298df4641/hypothesis-6.168.3-cp310-abi3-win32.whl", hash = "sha256:dc66390fb12d80585aa9222bf538ce8b7aa22cf5d118250647355a1c9e8f62f4", size = 678211, upload-time = "2026-09-28T05:19:23.355Z" }, + { url = "https://files.pythonhosted.org/packages/b0/db/0a02b146ad1c30f9716362f2e1a379dfd6b0f7be03ac1e22c9da24c9e2a9/hypothesis-6.168.3-cp310-abi3-win_amd64.whl", hash = "sha256:92325b276360fe86c5bf71a568c0d53a6d140b0de36dcf029f9164a17803bb24", size = 684906, upload-time = "2026-09-28T05:18:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c0/958deaf726848f96f52250740bf39f13f476b068e22f73e2b358eaa07532/hypothesis-6.168.3-cp310-abi3-win_arm64.whl", hash = "sha256:3cf6f1eeaf41cd8d60cf1f88fde905ca1dd77c906929a507d6ac7f66f2ccba2a", size = 683292, upload-time = "2026-09-28T05:19:57.404Z" }, + { url = "https://files.pythonhosted.org/packages/0c/a2/6787da846d929e52fc3344d803299c45782dbeac287528af08380e984bf8/hypothesis-6.168.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b230a850de63334c16654a34a2d547e0179d36b9071d4439b3e7237f6d077e7", size = 793254, upload-time = "2026-09-28T05:19:36.294Z" }, + { url = "https://files.pythonhosted.org/packages/89/ba/2893f5ca42501f4d562cba3229c8994c3a3e5fac66ef60c1e0e0b5220a5d/hypothesis-6.168.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d63b0226cd3e0d8bdd97c3384b22a21934ed4d53246c1c93575dada616672499", size = 784699, upload-time = "2026-09-28T05:18:54.194Z" }, + { url = "https://files.pythonhosted.org/packages/b7/aa/7d7349daf75b71f6f35876f8de115e974c5c04d600e86df1b2779b0883ab/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0369f5df055f96e117ab12e5f249668ff144731ab5280bc7a205fdf81f990b89", size = 1123016, upload-time = "2026-09-28T05:20:36.372Z" }, + { url = "https://files.pythonhosted.org/packages/00/f0/7774e1ea072708ea46cb25c4aeee5f9978b6c271764809069e8a6789858e/hypothesis-6.168.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f076bcd0f77fdcdb7797826c879099573d03a02e228ebe649ea917081b962ac0", size = 1168989, upload-time = "2026-09-28T05:20:08.44Z" }, + { url = "https://files.pythonhosted.org/packages/a7/7d/113992abed9efbd7944e3a496a382da58152dce126fddf6419ff31a0a57d/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b823ba1fcec8da730f29316d010b06d3f7e0c3828dcf91020e24c55e7d24652a", size = 1298626, upload-time = "2026-09-28T05:19:40.986Z" }, + { url = "https://files.pythonhosted.org/packages/d5/65/3659fa5e733027e5b37a27486e57f4053535e40853d23501c422bb275043/hypothesis-6.168.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dd2849c269d674e4618590f3b48d443bd4c06b5aef3d8d869086c2e6d213d248", size = 1335184, upload-time = "2026-09-28T05:20:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/b4/6c/35aab2221b5ea65125340f236e1a765a9e9f3b28d89a389ea0033fed29f7/hypothesis-6.168.3-cp313-cp313-win_amd64.whl", hash = "sha256:3ef7d26f5789e691401d5f87eafed9bd2763f0dbe47af6d6b66012a509404766", size = 682173, upload-time = "2026-09-28T05:19:12.762Z" }, + { url = "https://files.pythonhosted.org/packages/6e/79/27b0cb56ff5d2bc92458bf6fcebdb1f0dc01f57f043a178e9f9d74498169/hypothesis-6.168.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2d4c68729a13df9af4998d2652cfb5d541c5609b88c306880a5dfb284938ec2", size = 793287, upload-time = "2026-09-28T05:20:38.522Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/5342f95c3bc36586777ef8cb8eba87ac7acf0184210327fb0e6d8654e295/hypothesis-6.168.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b345f818083ec99966a43ca4f7b38feb62c6920ce28572bc2bde4948a8b7eaba", size = 784857, upload-time = "2026-09-28T05:20:42.609Z" }, + { url = "https://files.pythonhosted.org/packages/4a/5f/fc774241518a0588d362680a3084275058aab86d6bc02a2c61e1073142d0/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0608c610fc002978fc5de0f471770d8817e9455cb8649a47983c9f8bccfc1897", size = 1123330, upload-time = "2026-09-28T05:19:29.735Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e6/131f16775a3dca5f4fe27f0d6ad9a9851600098a54a872d163f6ebf0b664/hypothesis-6.168.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9dcf6448b1ecc37f2b23f2d1b3ddfc9ff6b6910614c15a642dc82f419b96037b", size = 1169178, upload-time = "2026-09-28T05:19:53.436Z" }, + { url = "https://files.pythonhosted.org/packages/06/61/c9f5bff8b73321c9fd12f2d69666c6e4861b97b60f5f24fdd6ac293007fd/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f9f094041dcce5119ebd7bab71062743ab02b6e654d056b370beda78c19e2", size = 1299127, upload-time = "2026-09-28T05:20:14.486Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/d4618f8ab12dd76c4d58ea052252759aa2bf0c4e71f4b502ba5d577e0b9f/hypothesis-6.168.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d479985fe73af97badfb72dc6d20c6a353e486a36f6d065f600026e0cf954a86", size = 1335390, upload-time = "2026-09-28T05:20:10.326Z" }, + { url = "https://files.pythonhosted.org/packages/95/b6/727161e17cc297df1aeea945de05c532033b66751625bec192bc5b5ff707/hypothesis-6.168.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:785e2c45f8c08e274b4bf1ccae97f1a4e09407e9a68e80790cfab1d17a4a45fa", size = 624291, upload-time = "2026-09-28T05:18:36.376Z" }, + { url = "https://files.pythonhosted.org/packages/77/23/2f9b506392a0b8712440bcb781abc5713be9ffac1303b6d10f3669050882/hypothesis-6.168.3-cp314-cp314-win_amd64.whl", hash = "sha256:320920b1e3dae8611eee8a03d063cf2187446f2a17c38cfb8a7fc1466f71eee2", size = 682064, upload-time = "2026-09-28T05:20:16.336Z" }, + { url = "https://files.pythonhosted.org/packages/50/93/efabfd95eb2b69c0c9fa1e1c83c8290aa2715d46baeb5a7764faf09c133a/hypothesis-6.168.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:35380baa981108a7f60c4eab71e46acd8d8f58440520346a1a6aba06dca7e074", size = 791875, upload-time = "2026-09-28T05:19:37.789Z" }, + { url = "https://files.pythonhosted.org/packages/1b/fc/2a0ada1623a9a33048bbf9f00a7c92513398c6b8865cba5efb854262ce30/hypothesis-6.168.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f0aaaed00438fa6673d856aaee12a4a8afbde82a9f3c5ff487cacfb064239afe", size = 783412, upload-time = "2026-09-28T05:18:48.266Z" }, + { url = "https://files.pythonhosted.org/packages/41/31/73c615d37eeb12da0209555612c32ced628d860267c23cfd2691c6207aea/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27e6df1576bf838e7f484d4ae5cba92114497ca30afb917056bf1ea33e95675", size = 1121662, upload-time = "2026-09-28T05:19:26.225Z" }, + { url = "https://files.pythonhosted.org/packages/92/54/1893344a3b8bbf83fa7c9bb7e96f6836580213854f46671cb749f7b4706c/hypothesis-6.168.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bc85014577982ec6d2e266edc5cd6e7a7d3674c648791ca983c67a52f89a0c4", size = 1167761, upload-time = "2026-09-28T05:18:49.85Z" }, + { url = "https://files.pythonhosted.org/packages/26/97/a61d82968febd25a0f08ffbbfabfaeead66f68fc5fa73f3c9ece1c36fc8f/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ccfc29505aa1cdcc254cb9cd701fd0811fef5480addd97d4127a00df023410f7", size = 1297309, upload-time = "2026-09-28T05:19:51.711Z" }, + { url = "https://files.pythonhosted.org/packages/bc/01/fe8e4cf6d39d02f4efadaa351427fa13722086cfc54a2da3579741c5177a/hypothesis-6.168.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:32d0699566aaa93f9e97a44705d7164386f1e91de78d277f53bada35e78bcc96", size = 1334260, upload-time = "2026-09-28T05:19:00.121Z" }, + { url = "https://files.pythonhosted.org/packages/39/6a/09177ebc62f4778f94379ecade6a51299d3d8c4f4b75ba636bf3a0202364/hypothesis-6.168.3-cp314-cp314t-win_amd64.whl", hash = "sha256:28d88fa174ecbd4ecbd7bb290f06d0db084a3971c3f511ae2830b65b5e25500f", size = 681992, upload-time = "2026-09-28T05:19:48.043Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f6/890bf33d608cd63348d3146a3ca83362fbf09e589c5e5230a00c403070f4/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:61f5782d807b1e6aa5c9beef1054cb2037e7d1add48a64972cd6ee447778781f", size = 791231, upload-time = "2026-09-28T05:18:43.998Z" }, + { url = "https://files.pythonhosted.org/packages/03/dd/fc36b204f8aa7437c676576d9437f0422de46aa8a729e6cd5bbb0224e759/hypothesis-6.168.3-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:7aacf3cf40c7ce8f9e4924d5b57bc0b068beafdfed2347bcf160a28b4cfbba7b", size = 783171, upload-time = "2026-09-28T05:19:20.353Z" }, + { url = "https://files.pythonhosted.org/packages/5c/d2/3cd087d577db67c404991d7723e64d7cd75570cdc82a0f7329aed6f2086c/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01768a03a30dc54df7fe457c0b34016c84d598fa00d5965ededab93ba4eb3408", size = 1121152, upload-time = "2026-09-28T05:18:39.095Z" }, + { url = "https://files.pythonhosted.org/packages/42/6f/a05a68cc66e7a22701c50bf94fb9ddeb36f4f5d0de3cc130066709eda621/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1a8a4ffc6c6e6e577f2bfbcfebf7cffbb310283ba2532c729570a0a752cebaaa", size = 1144069, upload-time = "2026-09-28T05:20:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/4a/f4/fdd7a093fb2fb3a0ce24256c92ad5bf67936f4669bd06d5ceae4298150ec/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b987d73eba95183a7e59cca6d1925c588aa4307922d9852cc2b4d282a2ae4128", size = 1146692, upload-time = "2026-09-28T05:20:25.83Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c8/6dbd4377e935505ae4fc8ee4c7b18c69994c4bbaca015447c470218bdbee/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4569c39bd97d9573e946429ed676f3b55a7c8ed80920addd67d384a47d59b38", size = 1189480, upload-time = "2026-09-28T05:20:46.734Z" }, + { url = "https://files.pythonhosted.org/packages/ba/bf/ff288b496b690000d2686dcfa7f67855e4c5a6dddb464ed8e4880be83bb2/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6042b8707a4b25bbbfe20b110258fb7549a68951e5525608a8ce06091039e9fc", size = 1167109, upload-time = "2026-09-28T05:19:24.758Z" }, + { url = "https://files.pythonhosted.org/packages/d9/d1/3fee2bc29fc747fabbb27e174515592e390809ca465d3bbbabbcfa3235a8/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:f413629de94d38a7a2ad259697a6752526143cb49c2e2d7bdba19381c80693f6", size = 1126823, upload-time = "2026-09-28T05:18:52.636Z" }, + { url = "https://files.pythonhosted.org/packages/58/6a/585294392fa6a6d9719446a042a777ba98ecca265d9e48cbedc10c88df0f/hypothesis-6.168.3-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f071737e4e775bebba07e319eb645a880d1e0186b4d24bad23255e849d86d483", size = 1155773, upload-time = "2026-09-28T05:20:51.268Z" }, + { url = "https://files.pythonhosted.org/packages/cd/4d/fa283ff79996debf1ff08593e01f8b773eb8211b19b3d6d5d491317a65bf/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:f59a3912858d0609c26054aa1c474847c797937f5e0e1c77f0d3f08032e50f09", size = 1296671, upload-time = "2026-09-28T05:18:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/98f9c2f628da5afabb6c3e8df5d2d299d2d50bf306b14a6e26d3954c4951/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f09c05a23a8025dd5cad07c2ed299a46778e72d49ff0dd5bf353bcc0f7dc1580", size = 1422001, upload-time = "2026-09-28T05:19:04.188Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c9/f2109ade29a7ec284d55bca328d784743e8b13e321576fd34ee7e65f99af/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_i686.whl", hash = "sha256:44ace770bda3a0301739fc5a413c764de790c739df1f1a7218dd049f8594d9f5", size = 1374182, upload-time = "2026-09-28T05:20:27.764Z" }, + { url = "https://files.pythonhosted.org/packages/ce/6d/e05d5f72441564a3bebc71fa155deafd0fd3b015d6014ec8a00edf6e42bb/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:03b131043608f94a2578a2a896a1a72092acb7079815eef5b8513b08447e0e62", size = 1278402, upload-time = "2026-09-28T05:20:56.332Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/6988a7f1f69c5c530687dce03a32c157e2d878e3a2e32af9269ce016b78a/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:e9784aca26eddfe99b03a0292320db742cc8a74200ef864fdce949f526f973cd", size = 1297770, upload-time = "2026-09-28T05:19:11.387Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/5bb82b60b836bd2329e1fe01ad94efc2b14dfa806f8e50ed14b795d78770/hypothesis-6.168.3-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:2b52ac363096232bebc2add117e9178d91f4248f4cdb919fd1026b5f86a4bb16", size = 1333977, upload-time = "2026-09-28T05:20:02.483Z" }, + { url = "https://files.pythonhosted.org/packages/8e/7d/d419841b8f65481ea1a50c4ba36f2670d48b8c7a51e4e8586089564cf7f6/hypothesis-6.168.3-cp315-abi3.abi3t-win32.whl", hash = "sha256:d28e3a6b511a74ce37df5274b51f36c2b274fea7365e00b16f0c81e22acd5957", size = 675394, upload-time = "2026-09-28T05:19:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d2/1de6a2ad100e44817f2e9a8e8ba3eeaf4769621f4f87f2d4966c4971fcc4/hypothesis-6.168.3-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7b9638789548361a57d984f56619ac694a914c328181d911271618409261ff4a", size = 681699, upload-time = "2026-09-28T05:19:09.957Z" }, + { url = "https://files.pythonhosted.org/packages/c0/79/be3370fd02734d6b1d950183ae9224580d19fa8d356b95348d60f1e3eeb7/hypothesis-6.168.3-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:65d78e4357ec48ed2c67825f06740ee3599be4cfe770a6092bed07108d679ac5", size = 679849, upload-time = "2026-09-28T05:19:06.884Z" }, +] + [[package]] name = "idna" version = "3.11" @@ -1364,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.56.16" +version = "1.57.4" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1392,6 +1461,7 @@ dependencies = [ dev = [ { name = "build" }, { name = "furo" }, + { name = "hypothesis" }, { name = "itables" }, { name = "l0-python" }, { name = "pytest" }, @@ -1410,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" }, { name = "microcalibrate", specifier = ">=0.18.0" }, From ee7df6a85ebc95a4323dabca793b5c4a12cb22dd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 4 Oct 2026 06:54:41 -0400 Subject: [PATCH 2/4] Draw each wealth target's quantile independently microimpute's QRF draws every target's quantile from the same seed, so each household got one quantile for all eighteen targets. Chained on earlier draws, that made sparse targets near-certain for households drawn high for a related asset: on the raw 2024-25 FRS the imputed debt secured on UK land came to GBP 56.8bn across 0.19m households (WAS donor GBP 1.3bn, 0.03m), and non-residential property debt GBP 51.8bn (donor GBP 5.7bn). Each target now gets its own seed after fitting (recorded in the model metadata, so cached models retrain): land debt falls to GBP 7.0bn across 0.04m households, non-residential debt to GBP 6.9bn, and savings from GBP 1,890bn to GBP 733bn (donor GBP 457bn on 7% fewer households). docs/imputations.md: overseas property and shares are not exported separately but are inside the property and financial totals (review nit). Co-Authored-By: Claude Opus 5.5 --- docs/imputations.md | 4 +++- .../datasets/imputations/wealth.py | 18 ++++++++++++++++++ .../tests/test_wealth_split.py | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/imputations.md b/docs/imputations.md index f09374d11..627b1cf85 100644 --- a/docs/imputations.md +++ b/docs/imputations.md @@ -61,7 +61,9 @@ Imputes household wealth components using demographic and income predictors. The | `non_residential_property_secured_debt` | `DVBldDebtR8_sum` | Mortgages and loans secured on other buildings | | `owned_land_secured_debt` | `DVLUKDebtR8_sum` | Loans secured on UK land | -`corporate_wealth` is the exact sum of its three components on every household; private pension wealth is not part of it, because pension rights are disregarded capital in every means test. A secured debt is zero wherever the household holds none of the asset it is secured on. Land and property overseas (`DVLOSValR8_sum`), other real estate (`DVOPrValR8_sum`) and overseas shares (`DVFShOSVR8_aggr`) are not mapped to any output. +`corporate_wealth` is the exact sum of its three components on every household; private pension wealth is not part of it, because pension rights are disregarded capital in every means test. A secured debt is zero wherever the household holds none of the asset it is secured on. Land and property overseas (`DVLOSValR8_sum`), other real estate (`DVOPrValR8_sum`) and overseas shares (`DVFShOSVR8_aggr`) are not exported separately; `property_wealth` and the financial wealth totals include them. + +Each target draws its quantile with its own seed. microimpute otherwise uses one seed for every target, so a household drawn high for one asset is drawn high for every later target conditioned on it, which makes sparse targets such as the debt secured on land near-certain for the households that hold the asset. --- diff --git a/policyengine_uk_data/datasets/imputations/wealth.py b/policyengine_uk_data/datasets/imputations/wealth.py index 4134d2b47..07e1d986e 100644 --- a/policyengine_uk_data/datasets/imputations/wealth.py +++ b/policyengine_uk_data/datasets/imputations/wealth.py @@ -86,6 +86,15 @@ "owned_land_secured_debt": "owned_land", } +# microimpute draws each target's quantile from the same seed, so a household +# gets the same quantile for every target. Chained on earlier draws, that makes +# sparse targets near-certain for households drawn high for a related asset +# (nearly every imputed land holder had debt secured on the land). Each target +# gets its own seed so the draws are independent. +QUANTILE_DRAW_SEEDS = { + variable: 1_000 + index for index, variable in enumerate(IMPUTE_VARIABLES) +} + # WAS round 8 sources of the targets built from more than one column. Part of # the model metadata, so a change of definition retrains a cached model. DERIVED_TARGETS = { @@ -218,6 +227,7 @@ def generate_was_table(was: pd.DataFrame): "predictor_variables": tuple(PREDICTOR_VARIABLES), "impute_variables": tuple(IMPUTE_VARIABLES), "derived_targets": tuple(DERIVED_TARGETS.items()), + "quantile_draw_seeds": tuple(QUANTILE_DRAW_SEEDS.items()), } @@ -344,6 +354,13 @@ def derive_wealth_outputs(output_df: pd.DataFrame) -> pd.DataFrame: return output_df +def use_independent_quantile_draws(model: QRF) -> QRF: + """Give each imputed target its own quantile-draw seed.""" + for variable, seed in QUANTILE_DRAW_SEEDS.items(): + model.model.models[variable].seed = seed + return model + + def save_imputation_models(): """ Train and save wealth imputation model. @@ -365,6 +382,7 @@ def save_imputation_models(): was[PREDICTOR_VARIABLES], was[IMPUTE_VARIABLES], ) + use_independent_quantile_draws(wealth) wealth.save(get_wealth_model_path()) return wealth diff --git a/policyengine_uk_data/tests/test_wealth_split.py b/policyengine_uk_data/tests/test_wealth_split.py index 1af7402fb..248c1e818 100644 --- a/policyengine_uk_data/tests/test_wealth_split.py +++ b/policyengine_uk_data/tests/test_wealth_split.py @@ -183,3 +183,17 @@ def test_derived_outputs_hold_their_identities(output): for asset in wealth.SECURED_DEBT_ASSETS.values(): assert derived[asset].equals(output[asset]) pd.testing.assert_frame_equal(wealth.derive_wealth_outputs(derived), derived) + + +def test_each_target_draws_its_quantile_independently(): + """microimpute seeds every target alike; the wealth model must not.""" + from types import SimpleNamespace + + models = {v: SimpleNamespace(seed=42) for v in wealth.IMPUTE_VARIABLES} + model = SimpleNamespace(model=SimpleNamespace(models=models)) + wealth.use_independent_quantile_draws(model) + seeds = [models[v].seed for v in wealth.IMPUTE_VARIABLES] + assert len(set(seeds)) == len(seeds) + assert dict(wealth.get_wealth_model_metadata()["quantile_draw_seeds"]) == { + v: models[v].seed for v in wealth.IMPUTE_VARIABLES + } From 23d10279a2a7a2fd159942329edd006ca5aea398 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 12 Oct 2026 11:35:02 -0400 Subject: [PATCH 3/4] Test that the built dataset's wealth split loads into the engine On the built Enhanced FRS: corporate_wealth equals its components, each secured debt appears only with its asset, the locked policyengine-uk reads every new input at its stored weighted total, deducts each debt from the asset it is imputed for, and keeps corporate_wealth as the share-like capital block. The engine checks need the policyengine-uk releases carrying #1974 and #1969; this PR stays a draft until the lock moves to them (UK hub review of 2ad43a68). Co-Authored-By: Claude Opus 5.5 --- .../tests/test_wealth_split_dataset.py | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 policyengine_uk_data/tests/test_wealth_split_dataset.py diff --git a/policyengine_uk_data/tests/test_wealth_split_dataset.py b/policyengine_uk_data/tests/test_wealth_split_dataset.py new file mode 100644 index 000000000..e83570f78 --- /dev/null +++ b/policyengine_uk_data/tests/test_wealth_split_dataset.py @@ -0,0 +1,59 @@ +"""The built dataset carries the WAS wealth split, and the locked engine reads it. + +The engine checks need a policyengine-uk release with the share-like +components and share_like_capital (PolicyEngine/policyengine-uk#1974) and the +secured-debt inputs and reg 49 valuation (#1969). +""" + +import numpy as np +import pytest + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.datasets.imputations.wealth import ( + CORPORATE_WEALTH_COMPONENTS, + SECURED_DEBT_ASSETS, +) + +NEW_INPUTS = ( + "private_pension_wealth", + *CORPORATE_WEALTH_COMPONENTS, + "cash_isa", + *SECURED_DEBT_ASSETS, +) + + +def test_corporate_wealth_is_the_sum_of_its_components(enhanced_frs): + household = enhanced_frs.household + components = household[list(CORPORATE_WEALTH_COMPONENTS)].sum(axis=1) + assert np.abs(household.corporate_wealth - components).max() < 0.01 + + +def test_secured_debt_only_where_the_asset_is_held(enhanced_frs): + household = enhanced_frs.household + for debt, asset in SECURED_DEBT_ASSETS.items(): + assert (household.loc[household[asset] <= 0, debt] == 0).all(), debt + assert (household[debt] >= 0).all(), debt + + +@pytest.mark.parametrize("variable", NEW_INPUTS) +def test_the_engine_reads_each_new_input(baseline, enhanced_frs, variable): + assert variable in baseline.tax_benefit_system.variables, ( + f"the locked policyengine-uk has no {variable}" + ) + household = enhanced_frs.household + stored = (household[variable] * household.household_weight).sum() + simulated = baseline.calculate(variable, CURRENT_FRS_RELEASE.base_year).sum() + assert simulated == pytest.approx(stored, rel=1e-4) + + +def test_the_engine_values_capital_with_these_inputs(baseline): + from policyengine_uk.utils.capital_valuation import SECURED_DEBT + + # The engine deducts each imputed debt from the asset it is imputed for. + assert {debt: asset for asset, debt in SECURED_DEBT.items()} == SECURED_DEBT_ASSETS + year = CURRENT_FRS_RELEASE.base_year + # corporate_wealth is the components' exact sum here, so the capital + # block keeps it. + assert baseline.calculate("share_like_capital", year).sum() == pytest.approx( + baseline.calculate("corporate_wealth", year).sum(), rel=1e-6 + ) From e9460eaf82af72cc2c7af540efd507622ee0e776 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 13 Oct 2026 08:36:20 -0400 Subject: [PATCH 4/4] Drop the duplicate hypothesis dev dependency Main already pins hypothesis>=6.168.3; the unversioned entry this PR added before the merge duplicated it (review of 23d10279). Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 - uv.lock | 1 - 2 files changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bf523a609..fffc84a7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ dependencies = [ dev = [ "ruff>=0.9.0", "pytest", - "hypothesis", "torch", "l0-python>=0.4.0", "tables", diff --git a/uv.lock b/uv.lock index bbfd23919..dce9c8285 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,6 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, - { name = "hypothesis", marker = "extra == 'dev'" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.168.3" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" },