diff --git a/changelog.d/was-wealth-components.added.md b/changelog.d/was-wealth-components.added.md new file mode 100644 index 00000000..598d6c8f --- /dev/null +++ b/changelog.d/was-wealth-components.added.md @@ -0,0 +1 @@ +Export `private_pension_wealth`, `directly_held_shares`, `unit_and_investment_trusts`, `stocks_and_shares_isa` and `cash_isa` from the Wealth and Assets Survey, and impute debt secured on non-main property and land (`other_residential_property_secured_debt`, `non_residential_property_secured_debt`, `owned_land_secured_debt`). diff --git a/changelog.d/was-wealth-split.fixed.md b/changelog.d/was-wealth-split.fixed.md new file mode 100644 index 00000000..83bc292b --- /dev/null +++ b/changelog.d/was-wealth-split.fixed.md @@ -0,0 +1 @@ +Split private pension wealth out of `corporate_wealth`, which now holds only shares, unit and investment trusts and stocks and shares ISAs, and add buy-to-let property to `other_residential_property_value`. diff --git a/docs/imputations.md b/docs/imputations.md index 75fd1a56..6af802ec 100644 --- a/docs/imputations.md +++ b/docs/imputations.md @@ -19,9 +19,9 @@ The imputations are applied in this order (dependencies noted): ## Wealth Imputation -**Source:** Wealth and Assets Survey (WAS) Round 7 (2018-2020) +**Source:** Wealth and Assets Survey (WAS) Round 8 (April 2020 to March 2022), household file -Imputes household wealth components using demographic and income predictors. +Imputes household wealth components using demographic and income predictors. The targets are imputed as a chain: each is fitted on the predictors and every earlier target. ### Predictors | Variable | Description | @@ -39,18 +39,31 @@ Imputes household wealth components using demographic and income predictors. | `region` | UK region | ### Outputs -| Variable | Description | -|----------|-------------| -| `owned_land` | Value of owned land | -| `property_wealth` | Total property wealth | -| `corporate_wealth` | Shares, pensions, investment ISAs | -| `gross_financial_wealth` | Total financial assets | -| `net_financial_wealth` | Financial assets minus liabilities | -| `main_residence_value` | Value of main home | -| `other_residential_property_value` | Value of other properties | -| `non_residential_property_value` | Value of non-residential property | -| `savings` | Savings account balances | -| `num_vehicles` | Number of vehicles owned | +| Variable | WAS round 8 source | Description | +|----------|--------------------|-------------| +| `owned_land` | `DVLUKValR8_sum` | Value of land in the UK | +| `property_wealth` | `DVPropertyR8` | Gross value of all property, main residence included | +| `private_pension_wealth` | `totalpenr8_aggr` − `dvvaldbt_scaper8_aggr` | Private pension wealth other than current-employment defined benefit rights: defined contribution pots, retained rights, pensions in payment | +| `directly_held_shares` | `DVFShUKVR8_aggr` + `DVFESHARESR8_aggr` | UK shares and employee shares and options, held outside ISAs and pooled funds | +| `unit_and_investment_trusts` | `DVFCollVR8_aggr` | Unit trusts and investment trusts (one survey question) | +| `stocks_and_shares_isa` | `DVIISAVR8_aggr` | Stocks and shares (investment) ISAs | +| `corporate_wealth` | derived | `directly_held_shares` + `unit_and_investment_trusts` + `stocks_and_shares_isa`, summed after imputation | +| `gross_financial_wealth` | `HFINWR8_SUM` | Total financial assets | +| `net_financial_wealth` | `HFINWNTR8_Sum` | Financial assets minus liabilities | +| `main_residence_value` | `DVHValueR8` | Value of main home | +| `other_residential_property_value` | `DVHseValR8_sum` + `DVBltValR8_sum` | Second homes and buy-to-let property | +| `non_residential_property_value` | `DVBlDValR8_sum` | Other buildings, such as shops and garages | +| `savings` | `DVSaValR8_aggr` | Savings account balances (cash ISAs excluded) | +| `num_vehicles` | `vcarnr8` | Number of vehicles owned | +| `student_loan_balance` | `Tot_LosR8_aggr` − `Tot_los_exc_SLCR8_aggr` | Student Loans Company balance, allocated to people | +| `cash_isa` | `DVCISAVR8_aggr` | Cash ISAs | +| `other_residential_property_secured_debt` | `DVHseDebtR8_sum` + `DVBLtDebtR8_sum` | Mortgages and loans secured on second homes and buy-to-let property | +| `non_residential_property_secured_debt` | `DVBldDebtR8_sum` | Mortgages and loans secured on other buildings | +| `owned_land_secured_debt` | `DVLUKDebtR8_sum` | Loans secured on UK land | + +`corporate_wealth` is the exact sum of its three components on every household; private pension wealth is not part of it, because pension rights are disregarded capital in every means test. A secured debt is zero wherever the household holds none of the asset it is secured on. Land and property overseas (`DVLOSValR8_sum`), other real estate (`DVOPrValR8_sum`) and overseas shares (`DVFShOSVR8_aggr`) are not exported separately; `property_wealth` and the financial wealth totals include them. + +Each target draws its quantile with its own seed. microimpute otherwise uses one seed for every target, so a household drawn high for one asset is drawn high for every later target conditioned on it, which makes sparse targets such as the debt secured on land near-certain for the households that hold the asset. --- diff --git a/policyengine_uk_data/datasets/imputations/wealth.py b/policyengine_uk_data/datasets/imputations/wealth.py index 36c5fd4d..07e1d986 100644 --- a/policyengine_uk_data/datasets/imputations/wealth.py +++ b/policyengine_uk_data/datasets/imputations/wealth.py @@ -45,10 +45,17 @@ "region", ] +# Chain order matters: each target is fitted on the predictors plus every +# earlier target. Private pension wealth takes the slot the old pension-laden +# corporate_wealth held, the share-like components follow it, and each +# secured debt comes after the asset it is secured on. IMPUTE_VARIABLES = [ "owned_land", "property_wealth", - "corporate_wealth", + "private_pension_wealth", + "directly_held_shares", + "unit_and_investment_trusts", + "stocks_and_shares_isa", "gross_financial_wealth", "net_financial_wealth", "main_residence_value", @@ -57,17 +64,67 @@ "savings", "num_vehicles", "student_loan_balance", + "cash_isa", + "other_residential_property_secured_debt", + "non_residential_property_secured_debt", + "owned_land_secured_debt", ] +# corporate_wealth is not imputed: it is the sum of its imputed components, +# so the identity holds on every household. +CORPORATE_WEALTH_COMPONENTS = ( + "directly_held_shares", + "unit_and_investment_trusts", + "stocks_and_shares_isa", +) + +# Debt secured on each capital asset (UC Regs 2013 reg. 49(1)(b) and its +# legacy equivalents deduct it from that asset's value only). +SECURED_DEBT_ASSETS = { + "other_residential_property_secured_debt": "other_residential_property_value", + "non_residential_property_secured_debt": "non_residential_property_value", + "owned_land_secured_debt": "owned_land", +} + +# microimpute draws each target's quantile from the same seed, so a household +# gets the same quantile for every target. Chained on earlier draws, that makes +# sparse targets near-certain for households drawn high for a related asset +# (nearly every imputed land holder had debt secured on the land). Each target +# gets its own seed so the draws are independent. +QUANTILE_DRAW_SEEDS = { + variable: 1_000 + index for index, variable in enumerate(IMPUTE_VARIABLES) +} + +# WAS round 8 sources of the targets built from more than one column. Part of +# the model metadata, so a change of definition retrains a cached model. +DERIVED_TARGETS = { + "private_pension_wealth": ("totalpenr8_aggr", "-dvvaldbt_scaper8_aggr"), + "directly_held_shares": ("DVFShUKVR8_aggr", "DVFESHARESR8_aggr"), + "other_residential_property_value": ("DVHseValR8_sum", "DVBltValR8_sum"), + "other_residential_property_secured_debt": ( + "DVHseDebtR8_sum", + "DVBLtDebtR8_sum", + ), + "student_loan_balance": ("Tot_LosR8_aggr", "-Tot_los_exc_SLCR8_aggr"), +} + WAS_RENAMES = { "R8xshhwgt": "household_weight", # Components for estimating land holdings. "DVLUKValR8_sum": "owned_land", # In the UK. + "DVLUKDebtR8_sum": "owned_land_secured_debt", "DVPropertyR8": "property_wealth", + # UK shares (listed or not) and employee shares and options, held outside + # ISAs and pooled funds. "DVFESHARESR8_aggr": "emp_shares_options", "DVFShUKVR8_aggr": "uk_shares", - "DVIISAVR8_aggr": "investment_isas", - "DVFCollVR8_aggr": "unit_investment_trusts", + # Investment ISAs: the survey's stocks and shares ISA question. + "DVIISAVR8_aggr": "stocks_and_shares_isa", + "DVCISAVR8_aggr": "cash_isa", + # Unit trusts and investment trusts: one survey question. + "DVFCollVR8_aggr": "unit_and_investment_trusts", + # Total private pension wealth and its current-employment defined benefit + # part, both valued on the SCAPE basis. "totalpenr8_aggr": "pensions", "dvvaldbt_scaper8_aggr": "db_pensions", # Predictors for fusing to FRS. @@ -88,12 +145,17 @@ # Other columns for reference. "DVLOSValR8_sum": "non_uk_land", "HFINWNTR8_Sum": "net_financial_wealth", - "DVLUKDebtR8_sum": "uk_land_debt", "HFINWR8_SUM": "gross_financial_wealth", "TotalWlthR8": "wealth", "DVhvalueR8": "main_residence_value", - "DVHseValR8_sum": "other_residential_property_value", + # Gross values of property other than the main residence, and the + # mortgages and loans secured on each class. + "DVHseValR8_sum": "second_homes_value", + "DVHseDebtR8_sum": "second_homes_debt", + "DVBltValR8_sum": "buy_to_let_value", + "DVBLtDebtR8_sum": "buy_to_let_debt", "DVBlDValR8_sum": "non_residential_property_value", + "DVBldDebtR8_sum": "non_residential_property_secured_debt", "DVTotinc_bhcR8": "household_net_income", "DVSaValR8_aggr": "savings", "vcarnr8": "num_vehicles", @@ -142,16 +204,18 @@ def generate_was_table(was: pd.DataFrame): was["is_renting"] = was["is_renter"] == 1 - was["non_db_pensions"] = was.pensions - was.db_pensions - was["corporate_wealth"] = was[ - [ - "non_db_pensions", - "emp_shares_options", - "uk_shares", - "investment_isas", - "unit_investment_trusts", - ] - ].sum(axis=1) + # Private pension wealth other than current-employment defined benefit + # rights: pension rights are disregarded capital in every means test, so it + # is kept out of corporate_wealth. + was["private_pension_wealth"] = was.pensions - was.db_pensions + was["directly_held_shares"] = was.uk_shares + was.emp_shares_options + was["corporate_wealth"] = was[list(CORPORATE_WEALTH_COMPONENTS)].sum(axis=1) + was["other_residential_property_value"] = ( + was.second_homes_value + was.buy_to_let_value + ) + was["other_residential_property_secured_debt"] = ( + was.second_homes_debt + was.buy_to_let_debt + ) was["student_loan_balance"] = was["total_loans"] - was["total_loans_exc_slc"] was["region"] = was["region"].map(REGIONS) return was @@ -162,6 +226,8 @@ def generate_was_table(was: pd.DataFrame): "was_household_tab_filename": CURRENT_WAS_RELEASE.household_tab_filename, "predictor_variables": tuple(PREDICTOR_VARIABLES), "impute_variables": tuple(IMPUTE_VARIABLES), + "derived_targets": tuple(DERIVED_TARGETS.items()), + "quantile_draw_seeds": tuple(QUANTILE_DRAW_SEEDS.items()), } @@ -269,6 +335,32 @@ def _allocate_student_loan_balance_to_people( return balances +def derive_wealth_outputs(output_df: pd.DataFrame) -> pd.DataFrame: + """ + Add the outputs derived from imputed targets. + + The share-like components are clipped at zero and summed into + corporate_wealth, so the identity holds on every household. A secured debt + is kept only where the household holds the asset it is secured on. + """ + output_df = output_df.copy() + for column in (*CORPORATE_WEALTH_COMPONENTS, *SECURED_DEBT_ASSETS): + output_df[column] = output_df[column].clip(lower=0) + output_df["corporate_wealth"] = output_df[list(CORPORATE_WEALTH_COMPONENTS)].sum( + axis=1 + ) + for debt, asset in SECURED_DEBT_ASSETS.items(): + output_df[debt] = output_df[debt].where(output_df[asset] > 0, 0.0) + return output_df + + +def use_independent_quantile_draws(model: QRF) -> QRF: + """Give each imputed target its own quantile-draw seed.""" + for variable, seed in QUANTILE_DRAW_SEEDS.items(): + model.model.models[variable].seed = seed + return model + + def save_imputation_models(): """ Train and save wealth imputation model. @@ -290,6 +382,7 @@ def save_imputation_models(): was[PREDICTOR_VARIABLES], was[IMPUTE_VARIABLES], ) + use_independent_quantile_draws(wealth) wealth.save(get_wealth_model_path()) return wealth @@ -338,7 +431,7 @@ def impute_wealth(dataset: UKSingleYearDataset) -> UKSingleYearDataset: input_df["region"] = input_df["region"].replace( "NORTHERN_IRELAND", "WALES" ) # WAS doesn't sample NI -> put NI households in Wales (closest aggregate) - output_df = model.predict(input_df) + output_df = derive_wealth_outputs(model.predict(input_df)) for column in output_df.columns: if column == "student_loan_balance": diff --git a/policyengine_uk_data/storage/uprating_factors.csv b/policyengine_uk_data/storage/uprating_factors.csv index 149ef19f..5b5bf690 100644 --- a/policyengine_uk_data/storage/uprating_factors.csv +++ b/policyengine_uk_data/storage/uprating_factors.csv @@ -6,6 +6,7 @@ bsp_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1 capital_gains,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 capital_gains_before_response,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 carers_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +cash_isa,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 child_benefit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 child_tax_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 childcare_expenses,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 @@ -13,11 +14,15 @@ clothing_and_footwear_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1 communication_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 corporate_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 diesel_spending,1.0,1.138,1.57,1.539,1.589,1.54,1.507,1.46,1.415,1.355,1.284,1.284,1.284,1.284,1.284 +directly_held_shares,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 dividend_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 domestic_energy_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 education_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 employee_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 employer_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 +non_residential_property_secured_debt,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +other_residential_property_secured_debt,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 +owned_land_secured_debt,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 pension_contributions_via_salary_sacrifice,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 employment_income,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 employment_income_before_lsr,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 @@ -59,6 +64,7 @@ pension_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384 personal_pension_contributions,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 petrol_spending,1.0,1.104,1.635,1.796,1.531,1.483,1.452,1.406,1.363,1.305,1.237,1.237,1.237,1.237,1.237 private_pension_income,1.0,1.003,1.053,1.106,1.161,1.216,1.261,1.288,1.315,1.346,1.346,1.346,1.346,1.346,1.346 +private_pension_wealth,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 private_transfer_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 property_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 recreation_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 @@ -72,9 +78,11 @@ state_pension_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1 statutory_maternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 statutory_paternity_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 statutory_sick_pay,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +stocks_and_shares_isa,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 student_loan_repayments,1.0,1.059,1.127,1.205,1.261,1.308,1.337,1.365,1.396,1.431,1.431,1.431,1.431,1.431,1.431 sublet_income,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 transport_consumption,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 +unit_and_investment_trusts,1.0,1.0,1.092,1.147,1.19,1.223,1.258,1.297,1.34,1.384,1.384,1.384,1.384,1.384,1.384 universal_credit_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 water_and_sewerage_charges,1.0,1.0,1.0,1.092,1.14,1.21,1.283,1.349,1.4,1.46,1.46,1.46,1.46,1.46,1.46 winter_fuel_allowance_reported,1.0,1.04,1.144,1.209,1.237,1.277,1.301,1.327,1.353,1.38,1.38,1.38,1.38,1.38,1.38 diff --git a/policyengine_uk_data/storage/uprating_growth_factors.csv b/policyengine_uk_data/storage/uprating_growth_factors.csv index 956f740d..b7037441 100644 --- a/policyengine_uk_data/storage/uprating_growth_factors.csv +++ b/policyengine_uk_data/storage/uprating_growth_factors.csv @@ -6,6 +6,7 @@ bsp_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0 capital_gains,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 capital_gains_before_response,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 carers_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +cash_isa,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 child_benefit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 child_tax_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 childcare_expenses,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 @@ -13,11 +14,15 @@ clothing_and_footwear_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0 communication_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 corporate_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 diesel_spending,0,0.138,0.379,-0.019,0.033,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 +directly_held_shares,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 dividend_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 domestic_energy_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 education_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 employee_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 employer_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 +non_residential_property_secured_debt,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +other_residential_property_secured_debt,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 +owned_land_secured_debt,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 pension_contributions_via_salary_sacrifice,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 employment_income,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 employment_income_before_lsr,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 @@ -59,6 +64,7 @@ pension_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0, personal_pension_contributions,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 petrol_spending,0,0.104,0.481,0.099,-0.147,-0.031,-0.021,-0.032,-0.031,-0.043,-0.052,0.0,0.0,0.0,0.0 private_pension_income,0,0.003,0.05,0.05,0.05,0.047,0.037,0.021,0.021,0.024,0.0,0.0,0.0,0.0,0.0 +private_pension_wealth,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 private_transfer_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 property_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 recreation_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 @@ -72,9 +78,11 @@ state_pension_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0 statutory_maternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 statutory_paternity_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 statutory_sick_pay,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +stocks_and_shares_isa,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 student_loan_repayments,0,0.059,0.064,0.069,0.046,0.037,0.022,0.021,0.023,0.025,0.0,0.0,0.0,0.0,0.0 sublet_income,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 transport_consumption,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 +unit_and_investment_trusts,0,0.0,0.092,0.05,0.037,0.028,0.029,0.031,0.033,0.033,0.0,0.0,0.0,0.0,0.0 universal_credit_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 water_and_sewerage_charges,0,0.0,0.0,0.092,0.044,0.061,0.06,0.051,0.038,0.043,0.0,0.0,0.0,0.0,0.0 winter_fuel_allowance_reported,0,0.04,0.1,0.057,0.023,0.032,0.019,0.02,0.02,0.02,0.0,0.0,0.0,0.0,0.0 diff --git a/policyengine_uk_data/tests/test_student_loan_balance.py b/policyengine_uk_data/tests/test_student_loan_balance.py index ea95bdf6..16000777 100644 --- a/policyengine_uk_data/tests/test_student_loan_balance.py +++ b/policyengine_uk_data/tests/test_student_loan_balance.py @@ -170,13 +170,12 @@ class DummyModel: @staticmethod def predict(_input_df): - return pd.DataFrame( - { - "owned_land": [10.0, 20.0], - "student_loan_balance": [900.0, 300.0], - }, - index=[1, 2], + output = pd.DataFrame( + 0.0, index=[1, 2], columns=list(wealth.IMPUTE_VARIABLES) ) + output["owned_land"] = [10.0, 20.0] + output["student_loan_balance"] = [900.0, 300.0] + return output class DummyMicrosimulation: def __init__(self, dataset): diff --git a/policyengine_uk_data/tests/test_wealth_split.py b/policyengine_uk_data/tests/test_wealth_split.py new file mode 100644 index 00000000..248c1e81 --- /dev/null +++ b/policyengine_uk_data/tests/test_wealth_split.py @@ -0,0 +1,199 @@ +"""Invariants of the WAS wealth split: pensions out of corporate_wealth, the +share-like components, buy-to-let property and secured debts.""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pandas as pd +from hypothesis import given, settings +from hypothesis import strategies as st + +_WEALTH_PATH = ( + Path(__file__).resolve().parents[1] / "datasets" / "imputations" / "wealth.py" +) +_WEALTH_SPEC = importlib.util.spec_from_file_location( + "wealth_split_module", _WEALTH_PATH +) +wealth = importlib.util.module_from_spec(_WEALTH_SPEC) +_WEALTH_SPEC.loader.exec_module(wealth) + +COMPONENTS = list(wealth.CORPORATE_WEALTH_COMPONENTS) +DEBTS = list(wealth.SECURED_DEBT_ASSETS) + +amount = st.floats(min_value=0, max_value=1e8, allow_nan=False) +signed_amount = st.floats(min_value=-1e7, max_value=1e8, allow_nan=False) + +WAS_SOURCE_COLUMNS = { + "pensions": "totalpenr8_aggr", + "db_pensions": "dvvaldbt_scaper8_aggr", + "uk_shares": "DVFShUKVR8_aggr", + "emp_shares": "DVFESHARESR8_aggr", + "isa": "DVIISAVR8_aggr", + "cash_isa": "DVCISAVR8_aggr", + "trusts": "DVFCollVR8_aggr", + "second_homes": "DVHseValR8_sum", + "second_homes_debt": "DVHseDebtR8_sum", + "btl": "DVBltValR8_sum", + "btl_debt": "DVBLtDebtR8_sum", + "buildings_debt": "DVBldDebtR8_sum", + "land_debt": "DVLUKDebtR8_sum", +} + + +def _was_row(**values): + row = {column: 0 for column in wealth.WAS_RENAMES} + row.update({"R8xshhwgt": 1, "GORR8": 11, "DVPriRntR8": 1}) + for key, value in values.items(): + row[WAS_SOURCE_COLUMNS[key]] = value + return row + + +@st.composite +def was_rows(draw): + pensions = draw(amount) + return _was_row( + pensions=pensions, + db_pensions=draw(st.floats(min_value=0, max_value=pensions)), + **{ + key: draw(amount) + for key in WAS_SOURCE_COLUMNS + if key not in ("pensions", "db_pensions") + }, + ) + + +def test_every_imputed_target_is_built_from_the_survey(): + was = wealth.generate_was_table(pd.DataFrame([_was_row()])) + missing = set(wealth.IMPUTE_VARIABLES) - set(was.columns) + assert not missing + + +def test_corporate_wealth_is_derived_not_imputed(): + assert "corporate_wealth" not in wealth.IMPUTE_VARIABLES + assert set(COMPONENTS) <= set(wealth.IMPUTE_VARIABLES) + assert "private_pension_wealth" in wealth.IMPUTE_VARIABLES + + +def test_each_secured_debt_is_imputed_after_its_asset(): + order = wealth.IMPUTE_VARIABLES + for debt, asset in wealth.SECURED_DEBT_ASSETS.items(): + assert order.index(asset) < order.index(debt) + + +def test_definitions_are_part_of_the_model_metadata(): + metadata = wealth.get_wealth_model_metadata() + assert dict(metadata["derived_targets"]) == wealth.DERIVED_TARGETS + + +def test_generate_was_table_maps_each_component(): + was = wealth.generate_was_table( + pd.DataFrame( + [ + _was_row( + pensions=500, + db_pensions=200, + uk_shares=10, + emp_shares=20, + isa=40, + cash_isa=7, + trusts=30, + second_homes=1_000, + second_homes_debt=100, + btl=2_000, + btl_debt=900, + buildings_debt=50, + land_debt=5, + ) + ] + ) + ).iloc[0] + assert was.private_pension_wealth == 300 + assert was.directly_held_shares == 30 + assert was.unit_and_investment_trusts == 30 + assert was.stocks_and_shares_isa == 40 + assert was.cash_isa == 7 + assert was.corporate_wealth == 100 + assert was.other_residential_property_value == 3_000 + assert was.other_residential_property_secured_debt == 1_000 + assert was.non_residential_property_secured_debt == 50 + assert was.owned_land_secured_debt == 5 + + +@settings(max_examples=200, deadline=None) +@given(was_rows()) +def test_split_conserves_the_old_corporate_wealth(row): + """Old corporate_wealth = new corporate_wealth + private_pension_wealth.""" + was = wealth.generate_was_table(pd.DataFrame([row])).iloc[0] + old_corporate_wealth = ( + row["totalpenr8_aggr"] + - row["dvvaldbt_scaper8_aggr"] + + row["DVFESHARESR8_aggr"] + + row["DVFShUKVR8_aggr"] + + row["DVIISAVR8_aggr"] + + row["DVFCollVR8_aggr"] + ) + assert np.isclose( + was.corporate_wealth + was.private_pension_wealth, + old_corporate_wealth, + rtol=1e-12, + atol=1e-6, + ) + assert np.isclose( + was.corporate_wealth, sum(was[c] for c in COMPONENTS), rtol=1e-12, atol=1e-6 + ) + assert was.private_pension_wealth >= 0 + assert ( + was.other_residential_property_value + == row["DVHseValR8_sum"] + row["DVBltValR8_sum"] + ) + + +@st.composite +def draws(draw): + n = draw(st.integers(min_value=1, max_value=20)) + columns = { + column: draw(st.lists(signed_amount, min_size=n, max_size=n)) + for column in (*COMPONENTS, *DEBTS, *wealth.SECURED_DEBT_ASSETS.values()) + } + # Exact zero assets must be reachable, since that is where debt is cleared. + for asset in wealth.SECURED_DEBT_ASSETS.values(): + zeros = draw(st.lists(st.booleans(), min_size=n, max_size=n)) + columns[asset] = [0.0 if z else v for z, v in zip(zeros, columns[asset])] + return pd.DataFrame(columns) + + +@settings(max_examples=300, deadline=None) +@given(draws()) +def test_derived_outputs_hold_their_identities(output): + derived = wealth.derive_wealth_outputs(output) + assert (derived[COMPONENTS] >= 0).all().all() + assert (derived[DEBTS] >= 0).all().all() + assert np.allclose( + derived.corporate_wealth, derived[COMPONENTS].sum(axis=1), rtol=0, atol=0 + ) + for debt, asset in wealth.SECURED_DEBT_ASSETS.items(): + no_asset = derived[asset] <= 0 + assert (derived.loc[no_asset, debt] == 0).all() + held = ~no_asset + assert np.array_equal( + derived.loc[held, debt], output.loc[held, debt].clip(lower=0) + ) + # Assets themselves are untouched, and deriving twice changes nothing. + for asset in wealth.SECURED_DEBT_ASSETS.values(): + assert derived[asset].equals(output[asset]) + pd.testing.assert_frame_equal(wealth.derive_wealth_outputs(derived), derived) + + +def test_each_target_draws_its_quantile_independently(): + """microimpute seeds every target alike; the wealth model must not.""" + from types import SimpleNamespace + + models = {v: SimpleNamespace(seed=42) for v in wealth.IMPUTE_VARIABLES} + model = SimpleNamespace(model=SimpleNamespace(models=models)) + wealth.use_independent_quantile_draws(model) + seeds = [models[v].seed for v in wealth.IMPUTE_VARIABLES] + assert len(set(seeds)) == len(seeds) + assert dict(wealth.get_wealth_model_metadata()["quantile_draw_seeds"]) == { + v: models[v].seed for v in wealth.IMPUTE_VARIABLES + } diff --git a/policyengine_uk_data/tests/test_wealth_split_dataset.py b/policyengine_uk_data/tests/test_wealth_split_dataset.py new file mode 100644 index 00000000..e83570f7 --- /dev/null +++ b/policyengine_uk_data/tests/test_wealth_split_dataset.py @@ -0,0 +1,59 @@ +"""The built dataset carries the WAS wealth split, and the locked engine reads it. + +The engine checks need a policyengine-uk release with the share-like +components and share_like_capital (PolicyEngine/policyengine-uk#1974) and the +secured-debt inputs and reg 49 valuation (#1969). +""" + +import numpy as np +import pytest + +from policyengine_uk_data.datasets.frs_release import CURRENT_FRS_RELEASE +from policyengine_uk_data.datasets.imputations.wealth import ( + CORPORATE_WEALTH_COMPONENTS, + SECURED_DEBT_ASSETS, +) + +NEW_INPUTS = ( + "private_pension_wealth", + *CORPORATE_WEALTH_COMPONENTS, + "cash_isa", + *SECURED_DEBT_ASSETS, +) + + +def test_corporate_wealth_is_the_sum_of_its_components(enhanced_frs): + household = enhanced_frs.household + components = household[list(CORPORATE_WEALTH_COMPONENTS)].sum(axis=1) + assert np.abs(household.corporate_wealth - components).max() < 0.01 + + +def test_secured_debt_only_where_the_asset_is_held(enhanced_frs): + household = enhanced_frs.household + for debt, asset in SECURED_DEBT_ASSETS.items(): + assert (household.loc[household[asset] <= 0, debt] == 0).all(), debt + assert (household[debt] >= 0).all(), debt + + +@pytest.mark.parametrize("variable", NEW_INPUTS) +def test_the_engine_reads_each_new_input(baseline, enhanced_frs, variable): + assert variable in baseline.tax_benefit_system.variables, ( + f"the locked policyengine-uk has no {variable}" + ) + household = enhanced_frs.household + stored = (household[variable] * household.household_weight).sum() + simulated = baseline.calculate(variable, CURRENT_FRS_RELEASE.base_year).sum() + assert simulated == pytest.approx(stored, rel=1e-4) + + +def test_the_engine_values_capital_with_these_inputs(baseline): + from policyengine_uk.utils.capital_valuation import SECURED_DEBT + + # The engine deducts each imputed debt from the asset it is imputed for. + assert {debt: asset for asset, debt in SECURED_DEBT.items()} == SECURED_DEBT_ASSETS + year = CURRENT_FRS_RELEASE.base_year + # corporate_wealth is the components' exact sum here, so the capital + # block keeps it. + assert baseline.calculate("share_like_capital", year).sum() == pytest.approx( + baseline.calculate("corporate_wealth", year).sum(), rel=1e-6 + ) diff --git a/pyproject.toml b/pyproject.toml index fffc84a7..bf523a60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ dev = [ "ruff>=0.9.0", "pytest", + "hypothesis", "torch", "l0-python>=0.4.0", "tables", diff --git a/uv.lock b/uv.lock index 5ae82135..bbfd2391 100644 --- a/uv.lock +++ b/uv.lock @@ -1433,7 +1433,7 @@ wheels = [ [[package]] name = "policyengine-uk-data" -version = "1.57.4" +version = "1.58.0" source = { editable = "." } dependencies = [ { name = "google-auth" }, @@ -1480,6 +1480,7 @@ requires-dist = [ { name = "google-auth" }, { name = "google-cloud-storage" }, { name = "huggingface-hub" }, + { name = "hypothesis", marker = "extra == 'dev'" }, { name = "hypothesis", marker = "extra == 'dev'", specifier = ">=6.168.3" }, { name = "itables", marker = "extra == 'dev'" }, { name = "l0-python", marker = "extra == 'dev'", specifier = ">=0.4.0" },