From 3cba3696760218ca58f69a34dc745155ebb66d77 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 14 Sep 2026 16:05:24 -0400 Subject: [PATCH 1/5] Separate SPM measurement scope from ordinary income --- changelog.d/spm-universe.breaking.md | 1 + docs/engineering/spm-universe.md | 107 ++++ docs/usage/microsimulation.md | 11 + policyengine_us/data/economic_assumptions.py | 18 +- .../household/cbo_means_tested_transfers.yaml | 2 +- .../gov/household/household_benefits.yaml | 4 +- .../gov/states/ca/cpuc/income_sources.yaml | 2 +- .../boost/boost_middle_class_tax_credit.py | 10 +- .../edaa_end_child_poverty_act.py | 6 +- .../congress/tlaib/end_child_poverty_act.py | 8 +- .../reforms/states/tx/rebate/tx_rebate.py | 4 +- policyengine_us/spm.py | 88 ++- policyengine_us/system.py | 4 +- policyengine_us/tests/core/test_spm_system.py | 11 +- .../tests/core/test_spm_universe.py | 521 ++++++++++++++++++ .../spm_unit_ordinary_housing_subsidy.yaml | 62 +++ .../income/household/household_benefits.py | 2 +- .../income/household/person_in_poverty.py | 3 +- .../income/spm_unit/in_deep_poverty.py | 5 +- .../household/income/spm_unit/in_poverty.py | 5 +- .../income/spm_unit/spm_unit_benefits.py | 4 +- .../spm_unit_capped_housing_subsidy.py | 21 +- .../spm_unit_is_in_deep_spm_poverty.py | 6 +- .../spm_unit/spm_unit_is_in_spm_poverty.py | 6 +- .../spm_unit_ordinary_housing_subsidy.py | 41 ++ ..._unit_ordinary_housing_subsidy_reported.py | 16 + .../spm_unit/spm_unit_spm_universe_status.py | 21 + 27 files changed, 954 insertions(+), 35 deletions(-) create mode 100644 changelog.d/spm-universe.breaking.md create mode 100644 docs/engineering/spm-universe.md create mode 100644 policyengine_us/tests/core/test_spm_universe.py create mode 100644 policyengine_us/tests/policy/baseline/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.yaml create mode 100644 policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.py create mode 100644 policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy_reported.py create mode 100644 policyengine_us/variables/household/income/spm_unit/spm_unit_spm_universe_status.py diff --git a/changelog.d/spm-universe.breaking.md b/changelog.d/spm-universe.breaking.md new file mode 100644 index 00000000000..d65203ca855 --- /dev/null +++ b/changelog.d/spm-universe.breaking.md @@ -0,0 +1 @@ +Require explicit SPM measurement-universe status in datasets, preserve missing SPM outcomes outside that universe, and return nullable floating-point poverty indicators. Separate ordinary housing income from SPM resources while preserving included units' existing housing cap and requiring a source-supported annual valuation for outside units with positive assistance. diff --git a/docs/engineering/spm-universe.md b/docs/engineering/spm-universe.md new file mode 100644 index 00000000000..7b31736abe3 --- /dev/null +++ b/docs/engineering/spm-universe.md @@ -0,0 +1,107 @@ +# SPM scope and ordinary housing income + +The country model separates Supplemental Poverty Measure (SPM) outputs from +ordinary income for records outside a dataset's declared measurement universe. +It preserves included units' existing housing cap and canonical calculator +amounts. Dataset construction must supply the measurement universe and any +required outside housing valuation. + +## Source decisions + +Supply `spm_unit_spm_universe_status` for every SPM unit and year: + +| Value | Measurement behavior | +| --- | --- | +| `INCLUDED` | Compute canonical amounts and poverty indicators. | +| `OUTSIDE` | Return missing SPM amounts, resources and poverty indicators; do not call the threshold provider for the unit. | +| `UNRESOLVED` | Raise `SPM_UNIVERSE_REQUIRED` when scope-dependent outputs are requested. | + +Dataset simulations default to `UNRESOLVED`. A household situation defaults to +`INCLUDED`. The model never infers scope from missing tenure, age, county or a +zero-adult measurement composition. A dataset must resolve scope for each year; +an earlier year's declaration does not supply a later year's declaration. +Automatic dataset extension drops these annual source declarations from its +generated future-year tables; explicitly supplied multi-year tables retain them. + +Source universes differ. Census's ACS research excludes all group quarters +because the public ACS does not identify the required subtypes. CPS-based SPM +includes some noninstitutional group quarters. A dataset release must identify +its source universe and validate household/member linkage before declaring +units outside. See [Census's ACS SPM research, pages 6–7](https://www.census.gov/content/dam/Census/library/working-papers/2020/demo/SEHSD-WP2020-09.pdf#page=6) +and [Census group-quarters guidance](https://www.census.gov/topics/income-poverty/guidance/group-quarters.html). + +## Housing in ordinary income + +`spm_unit_ordinary_housing_subsidy` supplies household benefits, CBO means-tested +transfers and California CPUC income. The corresponding contributed household +benefit overrides use the same output. SPM resources continue to use +`spm_unit_capped_housing_subsidy`. + +| Unit | Ordinary housing value | +| --- | --- | +| Included | Existing capped SPM housing value, including supported policy overrides. | +| Outside with finite, nonnegative assistance equal to zero | Zero. | +| Outside with positive assistance | Explicit `spm_unit_ordinary_housing_subsidy_reported`, between zero and assistance. | + +The reported input defaults to `-1`, a missing-value sentinel. Its year-local +formula and the dataset-extension exclusion prevent uprating or carry-forward +from manufacturing a new year's report. The model ignores this report for included units. A missing or +invalid required report raises `SPM_ORDINARY_HOUSING_VALUE_REQUIRED`. Invalid +assistance raises `SPM_HOUSING_ASSISTANCE_INVALID`. Core rejects explicit NaN +inputs before these formulas execute; computed invalid values reach these +typed checks. + +A release supplying an outside value must retain its unit identifier, year, +allocation and valuation source. Numeric bounds do not establish that evidence. +Likewise, accepting the country's computed/input assistance of zero does not +certify observed nonreceipt; the release still must validate its source inputs. + +Census first values housing assistance using market rent less tenant payment, +then caps the amount added to SPM resources at shelter need less tenant payment. +See the [SPM technical documentation, page 14](https://www2.census.gov/programs-surveys/supplemental-poverty-measure/datasets/spm/spm_techdoc.pdf#page=14). +That definition does not establish a housing valuation outside the SPM universe +or authorize changing CBO and CPUC conventions. This bridge preserves included +valuations and requires an explicit outside valuation. + +## Nullable indicators and aggregation + +The following outputs now use floating-point `0`, `1` and `NaN`: +`spm_unit_is_in_spm_poverty`, `spm_unit_is_in_deep_spm_poverty`, `in_poverty`, +`in_deep_poverty` and `person_in_poverty`. `NaN` identifies an outside record. +Included records with invalid resources or thresholds raise +`SPM_MEASUREMENT_INVALID`. +The indicators retain stock-quantity semantics: a monthly request returns the +annual classification without dividing a positive indicator by twelve. + +Use `calculate(..., map_to="person")` and MicroSeries `count`, `sum` and `mean` +for person-weighted summaries. Keep outside records in full-population coverage +counts and exclude their missing indicators from the eligible denominator. +An empty eligible denominator yields a missing rate; serialize it as JSON +`null`. Do not cast these indicators to `bool`, negate them with `~`, or use +them directly as Boolean selection masks. Compare an observed indicator with +`1` when selecting poor units and separately retain its missingness mask. + +Dataset inputs and `set_input` cannot override formula-owned measurement +outputs or the computed ordinary housing bridge. Preserve observed poverty +reports under separate report-only names. + +The country consumer audit found only direct alias projections in production +Python; usage documentation uses MicroSeries means. The historical Utah +notebook divides one poverty mean by another and requires an explicit empty or +zero-baseline check before reuse. Wrapper serialization and summary consumers +require the companion nullable-summary change. + +## Qualification limits + +Synthetic tests exercise mixed included/outside records, canonical-provider +selection, included parity, missing outcomes, housing consumers, year-local +reports and contributed reforms. They do not certify a native population build, +outside housing source values or a complete managed-wrapper run. + +Tiny country dataset fixtures also invoke dataset-dependent Medicaid allocation: +the model distributes statewide spending across the fixture's weighted enrolled +cost indices. Thus their absolute Medicaid totals do not represent household +estimates. Housing-component tests explicitly isolate Medicaid and CHIP where +needed to avoid float32 cancellation in CBO totals. This limitation reproduces +on the unchanged country base and requires representative population validation +before interpreting fiscal totals. diff --git a/docs/usage/microsimulation.md b/docs/usage/microsimulation.md index 95db177874a..866b34defaf 100644 --- a/docs/usage/microsimulation.md +++ b/docs/usage/microsimulation.md @@ -89,6 +89,11 @@ poverty_rate = person_poverty.mean() hh_eitc = sim.calc("eitc", map_to="household") ``` +SPM poverty indicators contain `0` and `1` for included records and `NaN` for +records outside the declared measurement universe. MicroSeries means exclude +missing indicators; an empty eligible population has a missing rate. Preserve +that rate as JSON `null`, and do not cast the indicators to Boolean values. + ## Available datasets ### The default build @@ -109,6 +114,12 @@ source-backed `is_spm_independent_minor_role` values, and it must not store formula-owned SPM outputs such as `spm_unit_spm_threshold`. Any observed Census measurement is retained under a separate report-only name. +The file must also declare `spm_unit_spm_universe_status` for every unit and +measurement year. Outside units with positive housing assistance require an +explicit ordinary housing valuation for that year. See the +[SPM universe contract](../engineering/spm-universe.md) for source requirements, +nullable outputs and ordinary-income behavior. + The legacy files under `hf://policyengine/policyengine-us-data/` predate that contract, so resource and poverty outputs are not available over them: diff --git a/policyengine_us/data/economic_assumptions.py b/policyengine_us/data/economic_assumptions.py index 5d9c1a44da4..0ee9b0338ea 100644 --- a/policyengine_us/data/economic_assumptions.py +++ b/policyengine_us/data/economic_assumptions.py @@ -14,6 +14,13 @@ "calibration.gov.cbo.income_by_source.adjusted_gross_income" ) +# Scope and outside housing valuations require a source decision for each year. +# Dataset extension must not turn a base-year declaration into a new report. +ANNUAL_SPM_SOURCE_DECLARATIONS = ( + "spm_unit_spm_universe_status", + "spm_unit_ordinary_housing_subsidy_reported", +) + MICRODATA_UPRATING_OVERRIDES = { "american_opportunity_credit": DEFAULT_MICRODATA_UPRATING, "cdcc_relevant_expenses": DEFAULT_MICRODATA_UPRATING, @@ -86,7 +93,8 @@ def extend_single_year_dataset( If ``end_year`` is not provided, it defaults to the latest year covered by the CPI-U parameter (gov.bls.cpi.cpi_u). - Variables without an uprating parameter are carried forward unchanged. + Variables without an uprating parameter are carried forward unchanged, + except annual SPM source declarations, which require explicit new-year inputs. """ if system is None: from policyengine_us.system import system as _system @@ -109,6 +117,14 @@ def extend_single_year_dataset( for year in range(start_year + 1, end_year + 1): next_year = dataset.copy(deep=False) next_year.time_period = str(year) + # The loader flattens columns from every entity table. Remove reserved + # declarations wherever the source stored them, including legacy layouts. + for table in next_year.tables: + table.drop( + columns=list(ANNUAL_SPM_SOURCE_DECLARATIONS), + errors="ignore", + inplace=True, + ) datasets.append(next_year) multi_year_dataset = USMultiYearDataset(datasets=datasets) diff --git a/policyengine_us/parameters/gov/household/cbo_means_tested_transfers.yaml b/policyengine_us/parameters/gov/household/cbo_means_tested_transfers.yaml index c988f66ede8..6487e19dcc7 100644 --- a/policyengine_us/parameters/gov/household/cbo_means_tested_transfers.yaml +++ b/policyengine_us/parameters/gov/household/cbo_means_tested_transfers.yaml @@ -4,7 +4,7 @@ values: - medicaid - chip - snap - - spm_unit_capped_housing_subsidy + - spm_unit_ordinary_housing_subsidy - ssi - tanf - free_school_meals diff --git a/policyengine_us/parameters/gov/household/household_benefits.yaml b/policyengine_us/parameters/gov/household/household_benefits.yaml index be0a6e69cf3..c656a576047 100644 --- a/policyengine_us/parameters/gov/household/household_benefits.yaml +++ b/policyengine_us/parameters/gov/household/household_benefits.yaml @@ -27,7 +27,7 @@ values: # Contributed. - basic_income - trump_dividend - - spm_unit_capped_housing_subsidy + - spm_unit_ordinary_housing_subsidy - household_state_benefits - household_health_benefits 2024-01-01: @@ -53,7 +53,7 @@ values: # Contributed. - basic_income - trump_dividend - - spm_unit_capped_housing_subsidy + - spm_unit_ordinary_housing_subsidy - household_state_benefits - commodity_supplemental_food_program - household_health_benefits diff --git a/policyengine_us/parameters/gov/states/ca/cpuc/income_sources.yaml b/policyengine_us/parameters/gov/states/ca/cpuc/income_sources.yaml index 2f43cdfca8a..e039403f32c 100644 --- a/policyengine_us/parameters/gov/states/ca/cpuc/income_sources.yaml +++ b/policyengine_us/parameters/gov/states/ca/cpuc/income_sources.yaml @@ -36,7 +36,7 @@ values: - pension_income - retirement_distributions # Housing subsidies - - spm_unit_capped_housing_subsidy + - spm_unit_ordinary_housing_subsidy # Military subsidies - veterans_benefits - military_service_income diff --git a/policyengine_us/reforms/congress/tlaib/boost/boost_middle_class_tax_credit.py b/policyengine_us/reforms/congress/tlaib/boost/boost_middle_class_tax_credit.py index f56b87f3bc9..238fedded03 100644 --- a/policyengine_us/reforms/congress/tlaib/boost/boost_middle_class_tax_credit.py +++ b/policyengine_us/reforms/congress/tlaib/boost/boost_middle_class_tax_credit.py @@ -1,4 +1,5 @@ from policyengine_us.model_api import * +from policyengine_us.spm import spm_universe_mask def create_boost_middle_class_tax_credit() -> Reform: @@ -72,7 +73,7 @@ def formula(household, period, parameters): "unemployment_compensation", # Contributed. "basic_income", - "spm_unit_capped_housing_subsidy", + "spm_unit_ordinary_housing_subsidy", "household_state_benefits", "household_head_start_benefits", ] @@ -80,7 +81,7 @@ def formula(household, period, parameters): BENEFITS = [ benefit for benefit in BENEFITS - if benefit != "spm_unit_capped_housing_subsidy" + if benefit != "spm_unit_ordinary_housing_subsidy" ] previous_benefits = add(household, period, BENEFITS) middle_class_credit = add( @@ -101,6 +102,7 @@ class spm_unit_benefits(Variable): unit = USD def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) BENEFITS = [ "social_security", "ssi", @@ -144,8 +146,8 @@ def formula(spm_unit, period, parameters): ) previous_benefits = add(spm_unit, period, BENEFITS) if p.administered_through_ssa is False: - return previous_benefits - return add(spm_unit, period, BENEFITS) + middle_class_tax_credit + return where(included, previous_benefits, np.nan) + return where(included, previous_benefits + middle_class_tax_credit, np.nan) class reform(Reform): def apply(self): diff --git a/policyengine_us/reforms/congress/tlaib/economic_dignity_for_all_agenda/edaa_end_child_poverty_act.py b/policyengine_us/reforms/congress/tlaib/economic_dignity_for_all_agenda/edaa_end_child_poverty_act.py index 728643b9908..91fb2bd3a7c 100644 --- a/policyengine_us/reforms/congress/tlaib/economic_dignity_for_all_agenda/edaa_end_child_poverty_act.py +++ b/policyengine_us/reforms/congress/tlaib/economic_dignity_for_all_agenda/edaa_end_child_poverty_act.py @@ -1,4 +1,5 @@ from policyengine_us.model_api import * +from policyengine_us.spm import spm_universe_mask def create_ecpa_only() -> Reform: @@ -100,7 +101,7 @@ def formula(household, period, parameters): BENEFITS = [ benefit for benefit in BENEFITS - if benefit != "spm_unit_capped_housing_subsidy" + if benefit != "spm_unit_ordinary_housing_subsidy" ] # Add ECPA child benefit @@ -116,6 +117,7 @@ class spm_unit_benefits(Variable): unit = USD def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) BENEFITS = [ "social_security", "ssi", @@ -154,7 +156,7 @@ def formula(spm_unit, period, parameters): if not parameters(period).gov.hud.abolition: BENEFITS.append("spm_unit_capped_housing_subsidy") - return add(spm_unit, period, BENEFITS) + return where(included, add(spm_unit, period, BENEFITS), np.nan) class income_tax_refundable_credits(Variable): value_type = float diff --git a/policyengine_us/reforms/congress/tlaib/end_child_poverty_act.py b/policyengine_us/reforms/congress/tlaib/end_child_poverty_act.py index f24d8e49ffc..6f3c4198b6d 100644 --- a/policyengine_us/reforms/congress/tlaib/end_child_poverty_act.py +++ b/policyengine_us/reforms/congress/tlaib/end_child_poverty_act.py @@ -1,4 +1,5 @@ from policyengine_us.model_api import * +from policyengine_us.spm import spm_universe_mask def create_end_child_poverty_act() -> Reform: @@ -108,7 +109,7 @@ def formula(household, period, parameters): "unemployment_compensation", # Contributed. "basic_income", - "spm_unit_capped_housing_subsidy", + "spm_unit_ordinary_housing_subsidy", "household_state_benefits", "household_head_start_benefits", "ecpa_child_benefit", @@ -117,7 +118,7 @@ def formula(household, period, parameters): BENEFITS = [ benefit for benefit in BENEFITS - if benefit != "spm_unit_capped_housing_subsidy" + if benefit != "spm_unit_ordinary_housing_subsidy" ] return add(household, period, BENEFITS) @@ -129,6 +130,7 @@ class spm_unit_benefits(Variable): unit = USD def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) BENEFITS = [ "social_security", "ssi", @@ -165,7 +167,7 @@ def formula(spm_unit, period, parameters): BENEFITS.append("aca_ptc") if not parameters(period).gov.hud.abolition: BENEFITS.append("spm_unit_capped_housing_subsidy") - return add(spm_unit, period, BENEFITS) + return where(included, add(spm_unit, period, BENEFITS), np.nan) class income_tax_refundable_credits(Variable): value_type = float diff --git a/policyengine_us/reforms/states/tx/rebate/tx_rebate.py b/policyengine_us/reforms/states/tx/rebate/tx_rebate.py index 2a6d7f163f9..51dc62d7825 100644 --- a/policyengine_us/reforms/states/tx/rebate/tx_rebate.py +++ b/policyengine_us/reforms/states/tx/rebate/tx_rebate.py @@ -13,6 +13,7 @@ """ from policyengine_us.model_api import * +from policyengine_us.spm import spm_universe_mask from policyengine_core.periods import period as period_ from policyengine_core.periods import instant @@ -50,6 +51,7 @@ class spm_unit_benefits(Variable): unit = USD def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) BENEFITS = [ "social_security", "ssi", @@ -123,7 +125,7 @@ def formula(spm_unit, period, parameters): BENEFITS.append("assigned_aca_ptc") if not parameters(period).gov.hud.abolition: BENEFITS.append("spm_unit_capped_housing_subsidy") - return add(spm_unit, period, BENEFITS) + return where(included, add(spm_unit, period, BENEFITS), np.nan) class reform(Reform): def apply(self): diff --git a/policyengine_us/spm.py b/policyengine_us/spm.py index bedc086632e..b4cd59a8805 100644 --- a/policyengine_us/spm.py +++ b/policyengine_us/spm.py @@ -11,15 +11,27 @@ from functools import lru_cache from inspect import signature +import numpy as np + from policyengine_core.simulations import Simulation as CoreSimulation from policyengine_core.taxbenefitsystems import TaxBenefitSystem from spm_calculator.errors import SPMInputError from spm_calculator.policyengine_adapter import ( - FORMULA_OWNED_INPUTS, + FORMULA_OWNED_INPUTS as CALCULATOR_FORMULA_OWNED_INPUTS, PolicyEngineSPMProvider, + build_policyengine_variables, ) from spm_calculator.rolling_forecast import load_forecast +FORMULA_OWNED_INPUTS = CALCULATOR_FORMULA_OWNED_INPUTS | { + "in_poverty", + "in_deep_poverty", + "person_in_poverty", + "deep_poverty_line", + "deep_poverty_gap", + "spm_unit_ordinary_housing_subsidy", +} + CONFIG_FIELDS = frozenset( { @@ -40,6 +52,80 @@ ) +def default_spm_universe_status(unit): + """Household requests describe included units; datasets must declare scope. + + Data origin is a scalar construction contract, not a demographic inference. + In particular, missing age or tenure never makes a unit outside the universe. + """ + values = unit.simulation.tax_benefit_system.variables[ + "spm_unit_spm_universe_status" + ].possible_values + status = values.UNRESOLVED if unit.simulation.is_over_dataset else values.INCLUDED + return unit.filled_array(status.index) + + +def spm_universe_mask(unit, period): + """Require a resolved source decision and return included unit positions.""" + status = unit("spm_unit_spm_universe_status", period).decode_to_str() + if np.any(status == "UNRESOLVED"): + raise SPMInputError( + "SPM_UNIVERSE_REQUIRED", + "Declare INCLUDED or OUTSIDE for every SPM unit from the source " + "measurement universe; unresolved units cannot be measured.", + ) + return status == "INCLUDED" + + +def scoped_spm_amount(unit, period, field): + """Keep outside amounts missing and never pass those units to the provider.""" + included = spm_universe_mask(unit, period) + amounts = masked_policyengine_amount(unit, period, field, included) + return np.where(included, amounts, np.nan) + + +def country_spm_variables(): + """Apply country-owned scope to the calculator's registered amount fields.""" + fields = { + "spm_unit_reference_spm_threshold": "reference_threshold", + "spm_unit_unadjusted_spm_threshold": "unadjusted_threshold", + "spm_unit_geographic_adjustment": "geographic_factor", + "spm_unit_spm_threshold": "threshold", + "spm_unit_spm_threshold_housing_portion": "housing_portion", + } + + def formula_for(field): + def formula(unit, period, parameters): + return scoped_spm_amount(unit, period, field) + + return formula + + variables = build_policyengine_variables() + for variable in variables: + if variable.__name__ in fields: + # The calculator creates fresh classes for each system. Only the + # selection changes; its provider still owns every final amount. + variable.formula = formula_for(fields[variable.__name__]) + return variables + + +def nullable_spm_indicator(unit, period, income, threshold): + """Return 0/1 for measured units, NaN outside, and reject invalid inputs.""" + included = spm_universe_mask(unit, period) + if ( + not np.isfinite(income[included]).all() + or not np.isfinite(threshold[included]).all() + or np.any(threshold[included] <= 0) + ): + raise SPMInputError( + "SPM_MEASUREMENT_INVALID", + "Included units require finite resources and positive finite thresholds.", + ) + result = np.full(included.shape, np.nan) + result[included] = income[included] < threshold[included] + return result + + def is_county_fips(value): """Accept only a five-digit county FIPS code. diff --git a/policyengine_us/system.py b/policyengine_us/system.py index 6b83b8b3acd..13f0f6acd3d 100644 --- a/policyengine_us/system.py +++ b/policyengine_us/system.py @@ -37,11 +37,11 @@ ) from typing import Annotated -from spm_calculator.policyengine_adapter import build_policyengine_variables from policyengine_us.spm import ( SPMSimulationMixin, clone_spm_system, create_spm_provider, + country_spm_variables, ) COUNTRY_DIR = Path(__file__).parent @@ -92,7 +92,7 @@ def __init__( ): super().__init__(entities, reform=reform) self.spm_forecast_provider = create_spm_provider(spm) - self.add_variables(*build_policyengine_variables()) + self.add_variables(*country_spm_variables()) self.load_parameters(COUNTRY_DIR / "parameters") self.add_abolition_parameters() self.parameters = set_irs_uprating_parameter(self.parameters) diff --git a/policyengine_us/tests/core/test_spm_system.py b/policyengine_us/tests/core/test_spm_system.py index 2b60bcd08ed..20970ec3528 100644 --- a/policyengine_us/tests/core/test_spm_system.py +++ b/policyengine_us/tests/core/test_spm_system.py @@ -417,7 +417,16 @@ def small_dataset(): } ), **{ - entity: pd.DataFrame({f"{entity}_id": [1, 2]}) + entity: pd.DataFrame( + { + f"{entity}_id": [1, 2], + **( + {"spm_unit_spm_universe_status": ["INCLUDED", "INCLUDED"]} + if entity == "spm_unit" + else {} + ), + } + ) for entity in groups if entity != "household" }, diff --git a/policyengine_us/tests/core/test_spm_universe.py b/policyengine_us/tests/core/test_spm_universe.py new file mode 100644 index 00000000000..0e7f82390e0 --- /dev/null +++ b/policyengine_us/tests/core/test_spm_universe.py @@ -0,0 +1,521 @@ +"""Synthetic mixed-universe contracts requiring the Python simulation API. + +YAML covers the ordinary-housing bridge's scalar arithmetic. These tests inspect +provider calls, missing outputs, independent entity membership, and preservation +of explicitly supplied source data; the YAML runner cannot express those checks. +No native population data, data build, or default dataset is used. +""" + +from importlib import import_module +import socket + +import h5py +import numpy as np +import pandas as pd +import pytest +import requests +from spm_calculator.errors import SPMInputError +from spm_calculator.policyengine_adapter import PolicyEngineSPMProvider + +from policyengine_us import Microsimulation, Simulation +from policyengine_us.data.dataset_schema import USSingleYearDataset +from policyengine_us.system import CountryTaxBenefitSystem + + +YEAR = 2024 +STATUS = "spm_unit_spm_universe_status" +REPORTED = "spm_unit_ordinary_housing_subsidy_reported" +ORDINARY_HOUSING = "spm_unit_ordinary_housing_subsidy" +POVERTY_INDICATORS = ( + "spm_unit_is_in_spm_poverty", + "spm_unit_is_in_deep_spm_poverty", + "in_poverty", + "in_deep_poverty", + "person_in_poverty", +) +MEASUREMENT_AMOUNTS = ( + "spm_unit_reference_spm_threshold", + "spm_unit_unadjusted_spm_threshold", + "spm_unit_spm_threshold", + "spm_unit_spm_threshold_housing_portion", + "spm_unit_geographic_adjustment", + "spm_unit_capped_housing_subsidy", + "spm_unit_benefits", + "spm_unit_net_income", + "poverty_line", + "deep_poverty_line", + "poverty_gap", + "deep_poverty_gap", +) + + +@pytest.fixture(autouse=True) +def no_population_data_reads(monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("Mixed SPM tests must not read population data") + + monkeypatch.setattr(requests.sessions.Session, "request", forbidden) + monkeypatch.setattr(socket.socket, "connect", forbidden) + monkeypatch.setattr(pd, "HDFStore", forbidden) + monkeypatch.setattr(h5py, "File", forbidden) + + +@pytest.fixture +def mixed_source(): + """One ordinary parent/child unit and one source-marked GQ child unit.""" + groups = ("household", "tax_unit", "spm_unit", "family") + return USSingleYearDataset( + person=pd.DataFrame( + { + "person_id": [101, 102, 301], + "age": [40, 8, 12], + "employment_income": [6_000.0, 0.0, 0.0], + "is_spm_independent_minor_role": [False, False, False], + "is_household_head": [True, False, False], + "is_tax_unit_head": [True, False, True], + "is_tax_unit_dependent": [False, True, False], + **{f"person_{entity}_id": [10, 10, 30] for entity in groups}, + "person_marital_unit_id": [101, 102, 301], + } + ), + household=pd.DataFrame( + { + "household_id": [10, 30], + "state_code": ["CA", "CA"], + # The outside record deliberately lacks SPM geography. + "county_fips": ["06037", ""], + "household_weight": [2.0, 5.0], + } + ), + tax_unit=pd.DataFrame({"tax_unit_id": [10, 30]}), + family=pd.DataFrame({"family_id": [10, 30]}), + marital_unit=pd.DataFrame({"marital_unit_id": [101, 102, 301]}), + spm_unit=pd.DataFrame( + { + "spm_unit_id": [10, 30], + STATUS: ["INCLUDED", "OUTSIDE"], + # Included source valuation must not replace the legacy cap. + REPORTED: [123.0, 30_000.0], + "spm_unit_tenure_type": ["RENTER", "RENTER"], + "is_eligible_for_housing_assistance": [True, True], + "takes_up_housing_assistance_if_eligible": [True, True], + "hud_hap": [30_000.0, 30_000.0], + "hud_ttp": [1_000.0, 1_000.0], + } + ), + time_period=YEAR, + ) + + +def _only_included(source): + """A separately built comparison fixture with the identical ordinary unit.""" + return USSingleYearDataset( + person=source.person.loc[source.person.person_household_id == 10].copy(), + household=source.household.loc[source.household.household_id == 10].copy(), + tax_unit=source.tax_unit.loc[source.tax_unit.tax_unit_id == 10].copy(), + spm_unit=source.spm_unit.loc[source.spm_unit.spm_unit_id == 10].copy(), + family=source.family.loc[source.family.family_id == 10].copy(), + marital_unit=source.marital_unit.loc[ + source.marital_unit.marital_unit_id.isin([101, 102]) + ].copy(), + time_period=YEAR, + ) + + +def _capture_provider(monkeypatch): + calls = [] + original = PolicyEngineSPMProvider.calculate_unit + + def capture(self, **kwargs): + calls.append(kwargs.copy()) + assert kwargs["adults"] >= 1, "Outside child-only unit reached SPM provider" + assert kwargs.get("county_fips") != "", "Outside geography reached provider" + return original(self, **kwargs) + + monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", capture) + return calls + + +@pytest.mark.parametrize("simulation_type", [Simulation, Microsimulation]) +def test_mixed_measurement_retains_included_values_and_missing_outside( + mixed_source, monkeypatch, simulation_type +): + calls = _capture_provider(monkeypatch) + simulation = simulation_type(dataset=mixed_source) + included = simulation_type(dataset=_only_included(mixed_source)) + for variable in (*MEASUREMENT_AMOUNTS, *POVERTY_INDICATORS): + actual = simulation.calculate(variable, YEAR, map_to="person") + expected = included.calculate(variable, YEAR, map_to="person") + assert np.issubdtype(actual.dtype, np.floating), variable + np.testing.assert_array_equal(actual[:2], expected, err_msg=variable) + assert np.isnan(actual[2]), f"{variable} outside must be missing, never false" + if simulation_type is Microsimulation and variable in POVERTY_INDICATORS: + # The two included people inherit household weight 2; the outside + # person's weight 5 must not enter the measurement denominator. + assert actual.count() == 4 + assert actual.mean() == expected.mean() + assert calls, "Included units must execute the real canonical provider" + assert {call["year"] for call in calls} == {YEAR} + assert {call["county_fips"] for call in calls} == {"06037"} + np.testing.assert_array_equal( + simulation.calculate("person_id", YEAR), [101, 102, 301] + ) + np.testing.assert_array_equal(simulation.calculate("spm_unit_size", YEAR), [2, 1]) + np.testing.assert_array_equal( + simulation.calculate("spm_unit_count_children", YEAR), [1, 1] + ) + + +def test_ordinary_bridge_preserves_cap_and_explicit_outside_housing_value( + mixed_source, monkeypatch +): + calls = _capture_provider(monkeypatch) + simulation = Microsimulation(dataset=mixed_source) + actual_assistance = simulation.calculate("housing_assistance", YEAR) + ordinary = simulation.calculate(ORDINARY_HOUSING, YEAR) + capped = simulation.calculate("spm_unit_capped_housing_subsidy", YEAR) + np.testing.assert_array_equal(actual_assistance, [30_000, 30_000]) + assert 123 < ordinary[0] < 30_000 + assert ordinary[0] == capped[0] + assert ordinary[1] == 30_000 + assert np.isnan(capped[1]) + assert calls + assert all(call["adults"] == 1 and call["children"] == 1 for call in calls) + + +def test_excluded_spm_measurement_does_not_require_an_ordinary_housing_valuation( + mixed_source, monkeypatch +): + source = mixed_source.copy() + source.spm_unit.drop(columns=[REPORTED], inplace=True) + _capture_provider(monkeypatch) + simulation = Microsimulation(dataset=source) + poverty = simulation.calculate("spm_unit_is_in_spm_poverty", YEAR, map_to="person") + assert poverty.count() == 4 + assert np.isnan(poverty.iloc[2]) + + +@pytest.mark.parametrize( + "variable", + [ + "household_benefits", + "household_net_income", + "cbo_household_means_tested_transfers", + "ca_cpuc_countable_income", + ], +) +def test_ordinary_consumers_keep_mixed_population_and_reported_housing_component( + mixed_source, monkeypatch, variable +): + _capture_provider(monkeypatch) + if variable == "cbo_household_means_tested_transfers": + # Dataset Medicaid allocates each state's spending over that dataset's + # enrollees. This tiny fixture therefore receives billions per person, + # as it does on unchanged 2.1.0, losing housing-dollar float32 precision. + # Fix health transfers only for this isolated housing-component check; + # it makes no claim about absolute CBO or Medicaid benefit amounts. + mixed_source.person["medicaid"] = [0.0, 0.0, 0.0] + mixed_source.person["chip"] = [0.0, 0.0, 0.0] + source_without_value = mixed_source.copy() + source_without_value.spm_unit[REPORTED] = [123.0, 0.0] + simulation = Microsimulation(dataset=mixed_source) + comparison = Microsimulation(dataset=source_without_value) + actual = simulation.calculate(variable, YEAR) + baseline = comparison.calculate(variable, YEAR) + assert len(actual) == 2, ( + "Ordinary resource consumers must retain outside households" + ) + assert np.isfinite(actual).all(), variable + assert actual[0] == baseline[0], variable + # Only the source valuation changes; actual HUD assistance remains fixed. + assert actual[1] - baseline[1] == pytest.approx(30_000, abs=0.01), variable + np.testing.assert_array_equal( + simulation.calculate("housing_assistance", YEAR), + comparison.calculate("housing_assistance", YEAR), + ) + + +@pytest.mark.parametrize("reported", [None, -1.0, float("inf"), 30_001.0]) +def test_positive_outside_housing_requires_a_valid_explicit_value( + mixed_source, reported +): + source = mixed_source.copy() + if reported is None: + source.spm_unit.drop(columns=[REPORTED], inplace=True) + else: + source.spm_unit[REPORTED] = [123.0, reported] + with pytest.raises(SPMInputError) as error: + Microsimulation(dataset=source).calculate(ORDINARY_HOUSING, YEAR) + assert error.value.code == "SPM_ORDINARY_HOUSING_VALUE_REQUIRED" + + +@pytest.mark.parametrize("assistance", [-1.0, float("inf")]) +def test_invalid_actual_housing_assistance_is_not_treated_as_known_zero( + mixed_source, assistance +): + source = mixed_source.copy() + source.spm_unit["hud_hap"] = [30_000.0, assistance] + with pytest.raises(SPMInputError) as error: + Microsimulation(dataset=source).calculate(ORDINARY_HOUSING, YEAR) + assert error.value.code == "SPM_HOUSING_ASSISTANCE_INVALID" + + +@pytest.mark.parametrize("variable", [REPORTED, "hud_hap"]) +def test_explicit_source_nan_is_rejected_before_ordinary_calculation( + mixed_source, variable +): + source = mixed_source.copy() + source.spm_unit[variable] = [30_000.0, np.nan] + # Core's input holder rejects NaN before the country formula can run. + with pytest.raises(ValueError, match="input contains NaN values"): + Microsimulation(dataset=source) + + +def test_calculated_nan_assistance_fails_typed_validation(mixed_source, monkeypatch): + simulation = Microsimulation(dataset=mixed_source) + original_calculate = simulation.calculate + + def calculate(variable_name, *args, **kwargs): + if variable_name == "housing_assistance": + return np.array([30_000.0, np.nan]) + return original_calculate(variable_name, *args, **kwargs) + + # A formula can yield NaN after Core's source-input checks have passed. + monkeypatch.setattr(simulation, "calculate", calculate) + with pytest.raises(SPMInputError) as error: + simulation.calculate(ORDINARY_HOUSING, YEAR) + assert error.value.code == "SPM_HOUSING_ASSISTANCE_INVALID" + + +@pytest.mark.parametrize("has_target_year_report", [False, True]) +def test_outside_housing_report_must_be_explicit_for_calculation_year( + mixed_source, has_target_year_report +): + simulation = Microsimulation(dataset=mixed_source) + target_year = YEAR + 1 + simulation.set_input(STATUS, target_year, ["OUTSIDE", "OUTSIDE"]) + simulation.set_input("hud_hap", target_year, [30_000.0, 30_000.0]) + simulation.set_input("hud_ttp", target_year, [1_000.0, 1_000.0]) + simulation.set_input( + "is_eligible_for_housing_assistance", target_year, [True, True] + ) + simulation.set_input( + "takes_up_housing_assistance_if_eligible", target_year, [True, True] + ) + if has_target_year_report: + simulation.set_input(REPORTED, target_year, [20_000.0, 25_000.0]) + np.testing.assert_array_equal( + simulation.calculate(ORDINARY_HOUSING, target_year), [20_000, 25_000] + ) + else: + with pytest.raises(SPMInputError) as error: + simulation.calculate(ORDINARY_HOUSING, target_year) + assert error.value.code == "SPM_ORDINARY_HOUSING_VALUE_REQUIRED" + + +@pytest.mark.parametrize("source_table", ["spm_unit", "household"]) +def test_source_universe_is_not_inferred_for_generated_future_year( + mixed_source, source_table +): + source = mixed_source.copy() + if source_table == "household": + # The accepted dataset loader flattens entity tables, so a declaration + # stored on this one-SPM-unit-per-household table is also supplied. + for variable in (STATUS, REPORTED): + source.household[variable] = source.spm_unit.pop(variable) + original_tables = [frame.copy(deep=True) for frame in source.tables] + simulation = Microsimulation(dataset=source) + assert list(simulation.calculate(STATUS, YEAR)) == ["INCLUDED", "OUTSIDE"] + np.testing.assert_array_equal(simulation.calculate(REPORTED, YEAR), [123, 30_000]) + future_status = simulation.calculate(STATUS, YEAR + 1) + assert list(future_status) == ["UNRESOLVED", "UNRESOLVED"] + np.testing.assert_array_equal(simulation.calculate(REPORTED, YEAR + 1), [-1, -1]) + with pytest.raises(SPMInputError): + simulation.calculate("spm_unit_spm_threshold", YEAR + 1) + for original, current in zip(original_tables, source.tables): + pd.testing.assert_frame_equal(original, current) + + +@pytest.mark.parametrize("earnings,expected_poor", [(0.0, 1.0), (100_000.0, 0.0)]) +def test_monthly_poverty_status_preserves_annual_boolean_values_and_missingness( + mixed_source, earnings, expected_poor +): + source = mixed_source.copy() + # A single included adult with zero or high earnings makes both ordinary + # and deep-poverty outcomes unambiguous, alongside the outside child. + source.person = source.person.loc[source.person.person_id != 102].copy() + source.marital_unit = source.marital_unit.loc[ + source.marital_unit.marital_unit_id != 102 + ].copy() + source.person["employment_income"] = [earnings, 0.0] + source.spm_unit["hud_hap"] = [0.0, 30_000.0] + simulation = Microsimulation(dataset=source) + for variable in POVERTY_INDICATORS: + annual = simulation.calculate(variable, YEAR, map_to="person") + monthly = simulation.calculate(variable, f"{YEAR}-01", map_to="person") + assert annual.iloc[0] == expected_poor, variable + np.testing.assert_array_equal(monthly, annual, err_msg=variable) + assert np.isnan(monthly.iloc[1]), variable + assert monthly.count() == annual.count() == 2 + assert monthly.mean() == annual.mean() == expected_poor + + +def test_known_zero_outside_assistance_needs_no_reported_value_or_provider( + mixed_source, monkeypatch +): + source = mixed_source.copy() + source.spm_unit[STATUS] = ["OUTSIDE", "OUTSIDE"] + source.spm_unit["hud_hap"] = [0.0, 0.0] + source.spm_unit.drop(columns=[REPORTED], inplace=True) + + def forbidden(*args, **kwargs): + raise AssertionError("Known-zero outside assistance must not call SPM") + + monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", forbidden) + simulation = Microsimulation(dataset=source) + np.testing.assert_array_equal(simulation.calculate(ORDINARY_HOUSING, YEAR), [0, 0]) + + +def test_all_outside_measurements_have_no_provider_calls_or_poverty_denominator( + mixed_source, monkeypatch +): + source = mixed_source.copy() + source.spm_unit[STATUS] = ["OUTSIDE", "OUTSIDE"] + + def forbidden(*args, **kwargs): + raise AssertionError("An all-outside population must not execute SPM") + + monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", forbidden) + simulation = Microsimulation(dataset=source) + for variable in ("spm_unit_spm_threshold", *POVERTY_INDICATORS): + result = simulation.calculate(variable, YEAR, map_to="person") + assert np.isnan(result).all(), variable + assert result.count() == 0, variable + assert np.isnan(result.mean()), variable + + +@pytest.mark.parametrize("variable", ["spm_unit_spm_threshold", *POVERTY_INDICATORS]) +def test_unresolved_measurement_status_fails_closed(mixed_source, variable): + source = mixed_source.copy() + # Make both units calculable so composition/geography errors cannot mask + # a missing check for the unresolved source classification. + source.person["age"] = [40, 8, 40] + source.household["county_fips"] = ["06037", "36061"] + source.spm_unit[STATUS] = ["INCLUDED", "UNRESOLVED"] + with pytest.raises(SPMInputError): + Microsimulation(dataset=source).calculate(variable, YEAR) + + +def test_dataset_without_source_universe_defaults_to_unresolved(mixed_source): + source = mixed_source.copy() + source.person["age"] = [40, 8, 40] + source.household["county_fips"] = ["06037", "36061"] + source.spm_unit.drop(columns=[STATUS], inplace=True) + simulation = Microsimulation(dataset=source) + with pytest.raises(SPMInputError): + simulation.calculate("spm_unit_spm_threshold", YEAR) + + +def test_ordinary_household_situation_keeps_included_default(): + simulation = Simulation( + situation={ + "people": {"person1": {"age": {YEAR: 40}}}, + "households": { + "household1": {"members": ["person1"], "county_fips": {YEAR: "06037"}} + }, + } + ) + assert simulation.calculate("spm_unit_spm_threshold", YEAR)[0] > 0 + + +@pytest.mark.parametrize("variable", [*POVERTY_INDICATORS, ORDINARY_HOUSING]) +@pytest.mark.parametrize("ingress", ["dataset", "set_input"]) +def test_computed_poverty_and_housing_outputs_cannot_be_supplied_as_inputs( + mixed_source, variable, ingress +): + from policyengine_us.system import system + + entity = system.variables[variable].entity.key + source = mixed_source.copy() + frame = getattr(source, entity) + if ingress == "dataset": + frame[variable] = 1.0 + with pytest.raises(ValueError, match="formula-owned"): + Microsimulation(dataset=source) + else: + simulation = Microsimulation(dataset=source) + with pytest.raises(ValueError, match="formula-owned"): + simulation.set_input(variable, YEAR, np.ones(len(frame))) + + +def test_mixed_calculations_preserve_source_tables_and_membership(mixed_source): + before = [frame.copy(deep=True) for frame in mixed_source.tables] + simulation = Microsimulation(dataset=mixed_source) + simulation.calculate("household_net_income", YEAR) + simulation.calculate("spm_unit_is_in_spm_poverty", YEAR) + for original, current in zip(before, mixed_source.tables): + pd.testing.assert_frame_equal(original, current) + for entity in ("household", "tax_unit", "spm_unit", "family", "marital_unit"): + np.testing.assert_array_equal( + simulation.calculate(f"person_{entity}_id", YEAR), + mixed_source.person[f"person_{entity}_id"], + ) + + +@pytest.mark.parametrize( + "module_name,factory_name", + [ + pytest.param( + "congress.tlaib.boost.boost_middle_class_tax_credit", + "create_boost_middle_class_tax_credit", + id="boost", + ), + pytest.param( + "congress.tlaib.end_child_poverty_act", + "create_end_child_poverty_act", + id="ecpa", + ), + pytest.param( + "congress.tlaib.economic_dignity_for_all_agenda.edaa_end_child_poverty_act", + "create_ecpa_only", + id="edaa", + ), + pytest.param("states.tx.rebate.tx_rebate", "create_tx_rebate", id="tx-rebate"), + ], +) +@pytest.mark.parametrize("abolish_housing", [False, True]) +def test_contributed_benefit_overrides_preserve_universe_separation( + mixed_source, monkeypatch, module_name, factory_name, abolish_housing +): + _capture_provider(monkeypatch) + factory = getattr( + import_module(f"policyengine_us.reforms.{module_name}"), factory_name + ) + reform = ( + factory(), + {"gov.hud.abolition": {f"{YEAR}-01-01.{YEAR}-12-31": abolish_housing}}, + ) + system = CountryTaxBenefitSystem(reform=reform) + + def simulate(source): + return Microsimulation( + dataset=source, + tax_benefit_system=system, + ) + + simulation = simulate(mixed_source) + standalone = simulate(_only_included(mixed_source)) + measured = simulation.calculate("spm_unit_benefits", YEAR) + assert measured[0] == standalone.calculate("spm_unit_benefits", YEAR)[0] + # HUD abolition removes the capped housing term, so an explicit mask must + # still exclude outside resources even without a NaN from that term. + assert np.isnan(measured[1]) + ordinary = simulation.calculate("household_benefits", YEAR) + assert len(ordinary) == 2 + assert np.isfinite(ordinary).all() + source_without_value = mixed_source.copy() + source_without_value.spm_unit[REPORTED] = [123.0, 0.0] + comparison = simulate(source_without_value).calculate("household_benefits", YEAR) + assert ordinary[0] == comparison[0] + expected_component = 0 if abolish_housing else 30_000 + assert ordinary[1] - comparison[1] == pytest.approx(expected_component, abs=0.01) diff --git a/policyengine_us/tests/policy/baseline/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.yaml b/policyengine_us/tests/policy/baseline/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.yaml new file mode 100644 index 00000000000..16751fe0a32 --- /dev/null +++ b/policyengine_us/tests/policy/baseline/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.yaml @@ -0,0 +1,62 @@ +# The outside-universe amount is an explicit source valuation, not an inferred +# SPM cap or an automatic rule assigning the full HUD benefit to every GQ unit. +- name: Case 1, an included unit retains the canonical cap despite a source value. + period: 2024 + absolute_error_margin: 0.01 + input: + people: + person1: {age: 40} + person2: {age: 40} + person3: {age: 10} + person4: {age: 8} + spm_units: + spm_unit: + members: [person1, person2, person3, person4] + spm_unit_spm_universe_status: INCLUDED + spm_unit_ordinary_housing_subsidy_reported: 30_000 + spm_unit_tenure_type: RENTER + is_eligible_for_housing_assistance: true + takes_up_housing_assistance_if_eligible: true + hud_hap: 30_000 + hud_ttp: 10_000 + households: + household: + members: [person1, person2, person3, person4] + state_code: CA + county_fips: "06037" + output: + housing_assistance: 30_000 + # Existing canonical county 06037 housing portion less tenant payment. + spm_unit_ordinary_housing_subsidy: 17_015.922643864185 + +- name: Case 2, an outside child-only unit retains an explicit partial housing value. + period: 2024 + absolute_error_margin: 0.01 + input: + age: 12 + is_spm_independent_minor_role: false + spm_unit_spm_universe_status: OUTSIDE + spm_unit_ordinary_housing_subsidy_reported: 18_000 + is_eligible_for_housing_assistance: true + takes_up_housing_assistance_if_eligible: true + hud_hap: 30_000 + hud_ttp: 1_000 + state_code: CA + output: + housing_assistance: 30_000 + spm_unit_ordinary_housing_subsidy: 18_000 + +- name: Case 3, an outside unit with no assistance requires no housing valuation. + period: 2024 + absolute_error_margin: 0.01 + input: + age: 12 + is_spm_independent_minor_role: false + spm_unit_spm_universe_status: OUTSIDE + is_eligible_for_housing_assistance: true + takes_up_housing_assistance_if_eligible: true + hud_hap: 0 + state_code: CA + output: + housing_assistance: 0 + spm_unit_ordinary_housing_subsidy: 0 diff --git a/policyengine_us/variables/household/income/household/household_benefits.py b/policyengine_us/variables/household/income/household/household_benefits.py index 274b1892c61..4010677762f 100644 --- a/policyengine_us/variables/household/income/household/household_benefits.py +++ b/policyengine_us/variables/household/income/household/household_benefits.py @@ -14,6 +14,6 @@ def formula(household, period, parameters): BENEFITS = [ benefit for benefit in BENEFITS - if benefit != "spm_unit_capped_housing_subsidy" + if benefit != "spm_unit_ordinary_housing_subsidy" ] return add(household, period, BENEFITS) diff --git a/policyengine_us/variables/household/income/household/person_in_poverty.py b/policyengine_us/variables/household/income/household/person_in_poverty.py index 58baf5e2c38..01d111273ea 100644 --- a/policyengine_us/variables/household/income/household/person_in_poverty.py +++ b/policyengine_us/variables/household/income/household/person_in_poverty.py @@ -6,7 +6,8 @@ class person_in_poverty(Variable): documentation = "Whether person is in poverty" entity = Person definition_period = YEAR - value_type = bool + value_type = float + quantity_type = STOCK def formula(person, period, parameters): return person.spm_unit("in_poverty", period) diff --git a/policyengine_us/variables/household/income/spm_unit/in_deep_poverty.py b/policyengine_us/variables/household/income/spm_unit/in_deep_poverty.py index db7ff02c41f..847ee298642 100644 --- a/policyengine_us/variables/household/income/spm_unit/in_deep_poverty.py +++ b/policyengine_us/variables/household/income/spm_unit/in_deep_poverty.py @@ -2,11 +2,12 @@ class in_deep_poverty(Variable): - value_type = bool + value_type = float + quantity_type = STOCK entity = SPMUnit label = "in deep poverty" documentation = "Whether a household is in deep poverty." definition_period = YEAR def formula(spm_unit, period, parameters): - return spm_unit("deep_poverty_gap", period) > 0 + return spm_unit("spm_unit_is_in_deep_spm_poverty", period) diff --git a/policyengine_us/variables/household/income/spm_unit/in_poverty.py b/policyengine_us/variables/household/income/spm_unit/in_poverty.py index 120b371cdbe..9c6e1b42470 100644 --- a/policyengine_us/variables/household/income/spm_unit/in_poverty.py +++ b/policyengine_us/variables/household/income/spm_unit/in_poverty.py @@ -2,11 +2,12 @@ class in_poverty(Variable): - value_type = bool + value_type = float + quantity_type = STOCK entity = SPMUnit label = "in poverty" documentation = "Whether a household is in poverty." definition_period = YEAR def formula(spm_unit, period, parameters): - return spm_unit("poverty_gap", period) > 0 + return spm_unit("spm_unit_is_in_spm_poverty", period) diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_benefits.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_benefits.py index 7908b195991..06cfdc554e0 100644 --- a/policyengine_us/variables/household/income/spm_unit/spm_unit_benefits.py +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_benefits.py @@ -1,4 +1,5 @@ from policyengine_us.model_api import * +from policyengine_us.spm import spm_universe_mask class spm_unit_benefits(Variable): @@ -9,6 +10,7 @@ class spm_unit_benefits(Variable): unit = USD def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) BENEFITS = [ "social_security", "ssi", @@ -82,4 +84,4 @@ def formula(spm_unit, period, parameters): BENEFITS.append("assigned_aca_ptc") if not parameters(period).gov.hud.abolition: BENEFITS.append("spm_unit_capped_housing_subsidy") - return add(spm_unit, period, BENEFITS) + return where(included, add(spm_unit, period, BENEFITS), np.nan) diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_capped_housing_subsidy.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_capped_housing_subsidy.py index b24f0ecf20b..1b9c0a99468 100644 --- a/policyengine_us/variables/household/income/spm_unit/spm_unit_capped_housing_subsidy.py +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_capped_housing_subsidy.py @@ -1,5 +1,6 @@ from policyengine_us.model_api import * -from policyengine_us.spm import masked_policyengine_amount +from policyengine_us.spm import masked_policyengine_amount, spm_universe_mask +from spm_calculator.errors import SPMInputError class spm_unit_capped_housing_subsidy(Variable): @@ -11,15 +12,23 @@ class spm_unit_capped_housing_subsidy(Variable): reference = "https://www2.census.gov/programs-surveys/supplemental-poverty-measure/datasets/spm/spm_techdoc.pdf" def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) housing_assistance = spm_unit("housing_assistance", period).astype("float64") + if not np.isfinite(housing_assistance[included]).all() or np.any( + housing_assistance[included] < 0 + ): + raise SPMInputError( + "SPM_HOUSING_ASSISTANCE_INVALID", + "Included housing assistance must be finite and nonnegative.", + ) # Only a unit with housing assistance has anything to cap. Consulting # the canonical housing portion for the other units would demand SPM # geography and composition from resource and benefit consumers that # never use the measurement, so the cap is evaluated for assisted units - # alone; every other unit's capped subsidy is zero by construction. - assisted = housing_assistance > 0 + # alone. Other included units receive zero; outside units remain missing. + assisted = included & (housing_assistance > 0) if not assisted.any(): - return np.zeros_like(housing_assistance) + return np.where(included, 0, np.nan) # Apply the country-owned cap to the unrounded canonical housing amount; # the model stores this final benefit amount with one dtype conversion. housing_portion = masked_policyengine_amount( @@ -27,4 +36,6 @@ def formula(spm_unit, period, parameters): ) tenant_payment = spm_unit("hud_ttp", period).astype("float64") cap = max_(housing_portion - tenant_payment, 0) - return where(assisted, min_(housing_assistance, cap), 0) + return where( + included, where(assisted, min_(housing_assistance, cap), 0), np.nan + ) diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_deep_spm_poverty.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_deep_spm_poverty.py index cdba4149ca5..882d0fe092d 100644 --- a/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_deep_spm_poverty.py +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_deep_spm_poverty.py @@ -1,8 +1,10 @@ from policyengine_us.model_api import * +from policyengine_us.spm import nullable_spm_indicator class spm_unit_is_in_deep_spm_poverty(Variable): - value_type = bool + value_type = float + quantity_type = STOCK entity = SPMUnit label = "SPM unit in deep SPM poverty" definition_period = YEAR @@ -10,4 +12,4 @@ class spm_unit_is_in_deep_spm_poverty(Variable): def formula(spm_unit, period, parameters): income = spm_unit("spm_unit_net_income", period) poverty_threshold = spm_unit("spm_unit_spm_threshold", period) / 2 - return income < poverty_threshold + return nullable_spm_indicator(spm_unit, period, income, poverty_threshold) diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_spm_poverty.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_spm_poverty.py index c0cc3a3ef68..d1f2573a98a 100644 --- a/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_spm_poverty.py +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_is_in_spm_poverty.py @@ -1,8 +1,10 @@ from policyengine_us.model_api import * +from policyengine_us.spm import nullable_spm_indicator class spm_unit_is_in_spm_poverty(Variable): - value_type = bool + value_type = float + quantity_type = STOCK entity = SPMUnit label = "SPM unit in SPM poverty" definition_period = YEAR @@ -10,4 +12,4 @@ class spm_unit_is_in_spm_poverty(Variable): def formula(spm_unit, period, parameters): income = spm_unit("spm_unit_net_income", period) poverty_threshold = spm_unit("spm_unit_spm_threshold", period) - return income < poverty_threshold + return nullable_spm_indicator(spm_unit, period, income, poverty_threshold) diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.py new file mode 100644 index 00000000000..20e77131846 --- /dev/null +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy.py @@ -0,0 +1,41 @@ +from policyengine_us.model_api import * +from policyengine_us.spm import spm_universe_mask +from spm_calculator.errors import SPMInputError + + +class spm_unit_ordinary_housing_subsidy(Variable): + value_type = float + entity = SPMUnit + definition_period = YEAR + unit = USD + label = "Housing subsidy value in ordinary income aggregates" + reference = "https://www2.census.gov/programs-surveys/supplemental-poverty-measure/datasets/spm/spm_techdoc.pdf#page=14" + + def formula(spm_unit, period, parameters): + included = spm_universe_mask(spm_unit, period) + assistance = spm_unit("housing_assistance", period).astype("float64") + if not np.isfinite(assistance).all() or np.any(assistance < 0): + raise SPMInputError( + "SPM_HOUSING_ASSISTANCE_INVALID", + "Ordinary housing assistance must be finite and nonnegative.", + ) + outside_assisted = ~included & (assistance > 0) + reported = spm_unit("spm_unit_ordinary_housing_subsidy_reported", period) + values = reported[outside_assisted] + if ( + not np.isfinite(values).all() + or np.any(values < 0) + or np.any(values > assistance[outside_assisted]) + ): + raise SPMInputError( + "SPM_ORDINARY_HOUSING_VALUE_REQUIRED", + "Outside units with positive housing assistance require a " + "source-supported ordinary housing value between zero and " + "assistance, for the same unit and year. Retain its allocation " + "and valuation provenance with the dataset; SPM scope alone " + "does not establish that value.", + ) + # Preserve the existing ordinary valuation for included units. Outside + # values are explicitly supplied, never an invented SPM cap or uncap. + capped = spm_unit("spm_unit_capped_housing_subsidy", period) + return where(included, capped, where(outside_assisted, reported, 0)) diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy_reported.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy_reported.py new file mode 100644 index 00000000000..4d53a7e8f45 --- /dev/null +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_ordinary_housing_subsidy_reported.py @@ -0,0 +1,16 @@ +from policyengine_us.model_api import * + + +class spm_unit_ordinary_housing_subsidy_reported(Variable): + value_type = float + default_value = -1 + entity = SPMUnit + definition_period = YEAR + unit = USD + label = "Source-supported ordinary housing subsidy value outside SPM scope" + reference = "https://www2.census.gov/programs-surveys/supplemental-poverty-measure/datasets/spm/spm_techdoc.pdf#page=14" + + def formula(spm_unit, period, parameters): + # Each year requires its own source declaration. A formula prevents the + # generic USD-input uprating rule from manufacturing a later report. + return -1 diff --git a/policyengine_us/variables/household/income/spm_unit/spm_unit_spm_universe_status.py b/policyengine_us/variables/household/income/spm_unit/spm_unit_spm_universe_status.py new file mode 100644 index 00000000000..9cf5824d2ea --- /dev/null +++ b/policyengine_us/variables/household/income/spm_unit/spm_unit_spm_universe_status.py @@ -0,0 +1,21 @@ +from policyengine_us.model_api import * +from policyengine_us.spm import default_spm_universe_status + + +class SPMUniverseStatus(Enum): + INCLUDED = "Included in the declared SPM measurement universe" + OUTSIDE = "Outside the declared SPM measurement universe" + UNRESOLVED = "Measurement universe has not been resolved" + + +class spm_unit_spm_universe_status(Variable): + value_type = Enum + possible_values = SPMUniverseStatus + default_value = SPMUniverseStatus.UNRESOLVED + entity = SPMUnit + definition_period = YEAR + label = "SPM measurement universe status" + reference = "https://www.census.gov/content/dam/Census/library/working-papers/2020/demo/SEHSD-WP2020-09.pdf#page=6" + + def formula(spm_unit, period, parameters): + return default_spm_universe_status(spm_unit) From 468d5752d6ff470ab6da924c3e8ef417bf23d67c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 14 Sep 2026 16:15:57 -0400 Subject: [PATCH 2/5] Require missing-aware Microdf counts for SPM summaries --- changelog.d/spm-universe.breaking.md | 2 ++ pyproject.toml | 2 +- uv.lock | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/changelog.d/spm-universe.breaking.md b/changelog.d/spm-universe.breaking.md index d65203ca855..92203653556 100644 --- a/changelog.d/spm-universe.breaking.md +++ b/changelog.d/spm-universe.breaking.md @@ -1 +1,3 @@ Require explicit SPM measurement-universe status in datasets, preserve missing SPM outcomes outside that universe, and return nullable floating-point poverty indicators. Separate ordinary housing income from SPM resources while preserving included units' existing housing cap and requiring a source-supported annual valuation for outside units with positive assistance. + +Require Microdf 1.3.0 or later so weighted counts exclude missing SPM outcomes. diff --git a/pyproject.toml b/pyproject.toml index 9d8803a0fe4..66f74d1760c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ classifiers = [ "Topic :: Scientific/Engineering :: Information Analysis", ] dependencies = [ - "microdf-python>=1.0.0", + "microdf-python>=1.3.0", # pandas 3 copy-on-write enables the shallow-copy fast path in dataset # extension and is available on every supported Python version. "pandas>=3.0", diff --git a/uv.lock b/uv.lock index 5d86fcc7c44..cd908158223 100644 --- a/uv.lock +++ b/uv.lock @@ -1201,15 +1201,15 @@ wheels = [ [[package]] name = "microdf-python" -version = "1.2.1" +version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "pandas" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/96/6f9f37f79f2c6440d91036a7bf8111dd4b983c577a7e96d45bf3ca4171f3/microdf_python-1.2.1.tar.gz", hash = "sha256:d4f58e4e0c21decd0c6d425b115db8acc72751c558f48d2a1c3a6619f168a94a", size = 19641, upload-time = "2026-01-25T13:40:57.147Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/1b/a4b6a53c8e6668c190e1928ebf9f8c67d6a4995ff4a4b0e3403ded30d4ab/microdf_python-1.3.0.tar.gz", hash = "sha256:f352fe32c25a4fd6d6188ef6aaca0214b53b29a8a7913aca6af0362dd9c922bd", size = 28141, upload-time = "2026-04-17T23:20:57.018Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/2e/375ab71f8d91b691597247b186a4d7b156d2ed975dfb00450e560beae747/microdf_python-1.2.1-py3-none-any.whl", hash = "sha256:3c3d318a82cba7db0ef5a72e8a73a6072fe0bc7a9cb59b1eac01a26ee8c82e7c", size = 20879, upload-time = "2026-01-25T13:40:55.877Z" }, + { url = "https://files.pythonhosted.org/packages/b9/aa/ce692e0488b5e6c0943f52886cf6a15f5d9a6dee7744b190a3a1d67f5d6e/microdf_python-1.3.0-py3-none-any.whl", hash = "sha256:e17757a198648108eae24f39384e27b5ae8061acf1b43fa56520010dc00a7b55", size = 28908, upload-time = "2026-04-17T23:20:56.051Z" }, ] [[package]] @@ -1758,7 +1758,7 @@ requires-dist = [ { name = "coverage", marker = "extra == 'dev'", specifier = ">=7.9.2" }, { name = "furo", marker = "extra == 'dev'", specifier = ">=2024.8.6" }, { name = "jupyter-book", marker = "extra == 'dev'", specifier = ">=1.0.4.post1" }, - { name = "microdf-python", specifier = ">=1.0.0" }, + { name = "microdf-python", specifier = ">=1.3.0" }, { name = "pandas", specifier = ">=3.0" }, { name = "policyengine-core", specifier = ">=3.30.1" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, From 47e94fa491bbf65ae4b6bd5667f054445b26de36 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 14 Sep 2026 17:54:50 -0400 Subject: [PATCH 3/5] Align SPM tests with explicit measurement universe precedence --- .../tests/core/test_spm_universe.py | 19 +++++++++++++++---- .../tests/microsimulation/test_microsim.py | 15 ++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/policyengine_us/tests/core/test_spm_universe.py b/policyengine_us/tests/core/test_spm_universe.py index 0e7f82390e0..859f8d14662 100644 --- a/policyengine_us/tests/core/test_spm_universe.py +++ b/policyengine_us/tests/core/test_spm_universe.py @@ -394,16 +394,26 @@ def forbidden(*args, **kwargs): assert np.isnan(result.mean()), variable -@pytest.mark.parametrize("variable", ["spm_unit_spm_threshold", *POVERTY_INDICATORS]) -def test_unresolved_measurement_status_fails_closed(mixed_source, variable): +@pytest.mark.parametrize( + "variable", ["household_net_income", "spm_unit_spm_threshold", *POVERTY_INDICATORS] +) +def test_unresolved_measurement_status_fails_closed( + mixed_source, variable, monkeypatch +): source = mixed_source.copy() # Make both units calculable so composition/geography errors cannot mask # a missing check for the unresolved source classification. source.person["age"] = [40, 8, 40] source.household["county_fips"] = ["06037", "36061"] source.spm_unit[STATUS] = ["INCLUDED", "UNRESOLVED"] - with pytest.raises(SPMInputError): + + def forbidden(*args, **kwargs): + raise AssertionError("Unresolved measurement scope must fail before SPM") + + monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", forbidden) + with pytest.raises(SPMInputError) as error: Microsimulation(dataset=source).calculate(variable, YEAR) + assert error.value.code == "SPM_UNIVERSE_REQUIRED" def test_dataset_without_source_universe_defaults_to_unresolved(mixed_source): @@ -412,8 +422,9 @@ def test_dataset_without_source_universe_defaults_to_unresolved(mixed_source): source.household["county_fips"] = ["06037", "36061"] source.spm_unit.drop(columns=[STATUS], inplace=True) simulation = Microsimulation(dataset=source) - with pytest.raises(SPMInputError): + with pytest.raises(SPMInputError) as error: simulation.calculate("spm_unit_spm_threshold", YEAR) + assert error.value.code == "SPM_UNIVERSE_REQUIRED" def test_ordinary_household_situation_keeps_included_default(): diff --git a/policyengine_us/tests/microsimulation/test_microsim.py b/policyengine_us/tests/microsimulation/test_microsim.py index 7fa111a3754..ed09855498d 100644 --- a/policyengine_us/tests/microsimulation/test_microsim.py +++ b/policyengine_us/tests/microsimulation/test_microsim.py @@ -18,16 +18,13 @@ def test_legacy_cps_file_with_stored_measurements_is_rejected_at_load(): Microsimulation(dataset=CPS_2023) -def test_legacy_enhanced_cps_lacks_source_backed_spm_independence_roles(): +def test_legacy_enhanced_cps_requires_source_measurement_universe(): """enhanced_cps_2024 loads, but cannot produce SPM measurements. - Its county column does hold five-digit FIPS codes, so geography is not what - fails. 18 of its 43,134 SPM units are a lone 15-to-17-year-old carrying no - source-backed SPM independence role, so those units classify no measurement - adult, and every output that reaches the threshold - household net income, - benefits, poverty and marginal tax rates included - fails closed over the - file. Assert on the threshold itself: it raises straight off the - composition, without building the whole resource chain over 43,134 units. + Its five-digit county codes are intact, and 18 units independently lack a + source-backed measurement adult. The absent annual universe declaration + fails before composition or geography can be used for a threshold. Neither + those missing roles nor a valid county establishes the measurement scope. """ import numpy as np from spm_calculator.errors import SPMInputError @@ -43,7 +40,7 @@ def test_legacy_enhanced_cps_lacks_source_backed_spm_independence_roles(): assert (adults < 1).sum() == 18 with pytest.raises(SPMInputError) as error: simulation.calculate("spm_unit_spm_threshold", 2024) - assert error.value.code == "SPM_COMPOSITION_REQUIRED" + assert error.value.code == "SPM_UNIVERSE_REQUIRED" def test_county_persists_across_periods(): From 24fd92ee960c0d464ee41a99e000ec09ae44bf20 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 21 Sep 2026 22:41:12 -0400 Subject: [PATCH 4/5] Fail closed on an unreadable SPM measurement-universe declaration A dataset may store spm_unit_spm_universe_status as member indices rather than member names. Core's EnumArray.decode_to_str decodes an index outside the enum to "unknown" instead of rejecting it, and spm_universe_mask only rejected "UNRESOLVED", so a mis-encoded unit fell through to the outside branch: its SPM amounts, resources and poverty indicators went missing with no error, and it left the poverty denominator silently. That is exclusion inferred from unreadable data, which the scope declaration exists to prevent. Resolve scope only on the two declared decisions, so anything else raises SPM_UNIVERSE_REQUIRED. Cover both accepted storage encodings with a test asserting they select the same measured units. Co-Authored-By: Claude Opus 5 --- changelog.d/spm-universe.breaking.md | 2 + docs/engineering/spm-universe.md | 7 +++ policyengine_us/spm.py | 13 ++++- .../tests/core/test_spm_universe.py | 55 +++++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/changelog.d/spm-universe.breaking.md b/changelog.d/spm-universe.breaking.md index 6faecf39268..d44df907c82 100644 --- a/changelog.d/spm-universe.breaking.md +++ b/changelog.d/spm-universe.breaking.md @@ -1,3 +1,5 @@ Require explicit SPM measurement-universe status in datasets, keep SPM amounts, resources and poverty outcomes missing outside that universe, and return nullable floating-point poverty, equivalised-income and income-decile outputs. Require Microdf 1.3.0 or later so weighted counts exclude missing SPM outcomes. + +Fail closed when a dataset stores an SPM measurement-universe value the status enum does not define, instead of measuring that unit as outside the universe. diff --git a/docs/engineering/spm-universe.md b/docs/engineering/spm-universe.md index 65dcb4daca3..0d36190c682 100644 --- a/docs/engineering/spm-universe.md +++ b/docs/engineering/spm-universe.md @@ -14,6 +14,13 @@ Supply `spm_unit_spm_universe_status` for every SPM unit and year: | `OUTSIDE` | Return missing SPM amounts, resources and poverty indicators; do not call the threshold provider for the unit. | | `UNRESOLVED` | Raise `SPM_UNIVERSE_REQUIRED` when scope-dependent outputs are requested. | +Store the declaration on the SPM-unit table, for every year, as either the +member name (`"INCLUDED"`, `"OUTSIDE"`, `"UNRESOLVED"`) or the member index +(`0`, `1`, `2`); both encodings resolve the same scope. Any other stored value +raises `SPM_UNIVERSE_REQUIRED`: a name the enum does not define is rejected on +load, and an index the enum does not define fails closed at measurement rather +than removing the unit from the universe. + Dataset simulations default to `UNRESOLVED`. A household situation defaults to `INCLUDED`. The model never infers scope from missing tenure, age, county or a zero-adult measurement composition. A dataset must resolve scope for each year; diff --git a/policyengine_us/spm.py b/policyengine_us/spm.py index 31dbfcb6e19..73ef531c34e 100644 --- a/policyengine_us/spm.py +++ b/policyengine_us/spm.py @@ -120,13 +120,20 @@ def default_spm_universe_status(unit): def spm_universe_mask(unit, period): """Require a resolved source decision and return included unit positions.""" status = unit("spm_unit_spm_universe_status", period).decode_to_str() - if np.any(status == "UNRESOLVED"): + included = status == "INCLUDED" + # A dataset may store this enum as member indices, and core decodes an index + # outside the enum to "unknown" rather than rejecting it. Reading anything + # but a declaration as exclusion would silently drop a mis-encoded unit from + # the measurement, which is the inference from missing data this declaration + # exists to prevent, so only the two declared decisions resolve scope. + if not np.all(included | (status == "OUTSIDE")): raise SPMInputError( "SPM_UNIVERSE_REQUIRED", "Declare INCLUDED or OUTSIDE for every SPM unit from the source " - "measurement universe; unresolved units cannot be measured.", + "measurement universe; an unresolved or unreadable declaration " + "cannot be measured.", ) - return status == "INCLUDED" + return included def scoped_spm_amount(unit, period, field): diff --git a/policyengine_us/tests/core/test_spm_universe.py b/policyengine_us/tests/core/test_spm_universe.py index 82c12ba3311..44a296273fb 100644 --- a/policyengine_us/tests/core/test_spm_universe.py +++ b/policyengine_us/tests/core/test_spm_universe.py @@ -260,6 +260,61 @@ def forbidden(*args, **kwargs): assert error.value.code == "SPM_UNIVERSE_REQUIRED" +@pytest.mark.parametrize("variable", ALL_MEASUREMENTS) +def test_unreadable_measurement_status_fails_closed( + mixed_source, variable, monkeypatch +): + """An unreadable declaration is not a declaration of exclusion. + + A producer may store this enum as member indices rather than member names, + and core decodes an index outside the enum to ``"unknown"`` instead of + rejecting it (``EnumArray.decode_to_str``). Reading that as anything but a + failure would drop the unit from the measurement with no error, which is + exactly the exclusion-inferred-from-missing-data the scope declaration + exists to prevent. + """ + source = mixed_source.copy() + # Make both units calculable so composition/geography errors cannot mask + # a missing check for the unreadable source classification. + source.person["age"] = [40, 8, 40] + source.household["county_fips"] = ["06037", "36061"] + # One declared unit and one index the enum does not define. + source.spm_unit[STATUS] = np.array([0, 99], dtype=np.int8) + + def forbidden(*args, **kwargs): + raise AssertionError("An unreadable measurement scope must fail before SPM") + + monkeypatch.setattr(PolicyEngineSPMProvider, "calculate_unit", forbidden) + with pytest.raises(SPMInputError) as error: + Microsimulation(dataset=source).calculate(variable, YEAR) + assert error.value.code == "SPM_UNIVERSE_REQUIRED" + + +def test_member_index_declarations_resolve_the_same_scope_as_member_names( + mixed_source, +): + """Both accepted storage encodings of the declaration measure the same units. + + ``Enum.encode`` takes a string column by member name and an integer column + as member indices, so a producer may ship either. Neither encoding may + change which units are measured. + """ + by_name = mixed_source.copy() + by_name.spm_unit[STATUS] = ["INCLUDED", "OUTSIDE"] + by_index = mixed_source.copy() + by_index.spm_unit[STATUS] = np.array([0, 1], dtype=np.int8) + thresholds = [] + for source in (by_name, by_index): + simulation = Microsimulation(dataset=source) + assert list(simulation.calculate(STATUS, YEAR)) == ["INCLUDED", "OUTSIDE"] + thresholds.append( + np.asarray(simulation.calculate("spm_unit_spm_threshold", YEAR)) + ) + assert thresholds[0][0] > 0 + assert np.isnan(thresholds[0][1]) + np.testing.assert_array_equal(thresholds[0], thresholds[1]) + + def test_dataset_without_source_universe_defaults_to_unresolved(mixed_source): source = mixed_source.copy() source.person["age"] = [40, 8, 40] From 091cf350227754ed6e0e7985ed391e6e094c780e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 22 Sep 2026 00:37:27 -0400 Subject: [PATCH 5/5] Reconcile the merged future-poverty test with the nullable indicator test_future_poverty_in_all_states arrived from main asserting that spm_unit_is_in_spm_poverty has dtype bool. This branch makes the canonical poverty indicators nullable floats, so that assertion encodes the contract this branch replaces and the merge carried it in unreconciled. Assert the contract this branch does establish, and keep what the test was checking: the situation is wholly in-universe, so every state's unit must be an observed 0 or 1 and none may go missing. Co-Authored-By: Claude Opus 5 --- .../core/test_ssi_state_supplement_medicaid_dependency.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/policyengine_us/tests/core/test_ssi_state_supplement_medicaid_dependency.py b/policyengine_us/tests/core/test_ssi_state_supplement_medicaid_dependency.py index a64c80a17a7..595935c9db2 100644 --- a/policyengine_us/tests/core/test_ssi_state_supplement_medicaid_dependency.py +++ b/policyengine_us/tests/core/test_ssi_state_supplement_medicaid_dependency.py @@ -112,6 +112,11 @@ def test_future_poverty_in_all_states(year): }, ) poor = simulation.calculate("spm_unit_is_in_spm_poverty", year) - assert poor.dtype == np.dtype(bool) + # The canonical indicator is a nullable float now that SPM measurement scope + # is an explicit source decision: NaN marks a record outside the declared + # universe. A household situation is wholly in-universe, so every unit here + # is observed, and none of these states may go missing. + assert np.issubdtype(poor.dtype, np.floating) + assert np.isin(poor, [0.0, 1.0]).all() assert poor.size == len(groups) assert np.isfinite(simulation.calculate("spm_unit_benefits", year)).all()