From dc59f526fe3faff6a4bdcb164c212f6828fcf548 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 13:54:52 -0400 Subject: [PATCH 1/2] Split PUF E19200 residual into points, premiums and investment leaves Preserve the mortgage splitter and conserve the residual with nested complements. Propagate all three Person leaves through source, QRF, transfer, export and typed-spec contracts, with regenerated identities and coverage. Co-Authored-By: GPT-6.1 Sol (subfleet lane) --- ...9200-points-premiums-investment.changed.md | 7 + docs/concept-coverage/policyengine-us.md | 8 +- docs/evidence/spec-engine/us-f0-coverage.json | 158 +++--- docs/us-multispine-operator-ordering.md | 104 ++-- experiments/build_j_recert/buildj_base.sh | 2 + .../spec_engine/calibration_semantics.py | 18 +- .../build/spec_engine/field_usage.py | 28 +- .../build/spec_engine/imputation_semantics.py | 52 +- .../build/spec_engine/inventory_coverage.py | 86 ++-- .../schema/calibration.schema.json | 8 + .../spec_engine/schema/imputation.schema.json | 48 ++ .../spec_engine/schema/sources.schema.json | 17 +- .../build/us/ecps_parity_reference.json | 4 + .../us/release_input_coverage_manifest.json | 14 +- .../src/microcosm/build/us/source_stages.json | 13 +- .../src/microcosm/build/us/spec/battery.yaml | 10 + .../microcosm/build/us/spec/calibration.yaml | 2 + .../src/microcosm/build/us/spec/catalogs.yaml | 22 + .../microcosm/build/us/spec/imputation.yaml | 468 +++++++++++++----- .../src/microcosm/build/us/spec/sources.yaml | 15 +- .../microcosm/build/us_runtime/__init__.py | 6 +- .../build/us_runtime/fiscal_targets.py | 6 +- .../build/us_runtime/l0_refit_export.py | 5 +- .../us_runtime/puf_interest_components.py | 146 +++++- .../build/us_runtime/puf_qrf_chain.py | 7 +- .../microcosm/build/us_runtime/puf_support.py | 59 ++- .../us_runtime/release_input_coverage.py | 13 + .../build/us_runtime/stacked_spine.py | 30 +- .../us_runtime/us_late_overlap_ownership.py | 2 +- .../us_runtime/us_late_producer_registry.py | 10 +- .../us/test_imputation_lineage_spec.py | 10 +- .../us/test_spec_engine_coverage_tool.py | 18 +- ...spec_engine_stacked_authority_semantics.py | 4 +- .../us/test_spec_engine_typed_closure.py | 4 +- .../us/test_us_bundle_core_contracts.py | 8 +- .../engine_contract/us/test_us_spec_bundle.py | 9 +- .../shared/test_release_input_coverage.py | 26 + .../test_spec_engine_battery_semantics.py | 6 +- .../test_spec_engine_calibration_semantics.py | 47 +- .../shared/test_spec_engine_compiler_ir.py | 6 +- .../shared/test_spec_engine_field_usage.py | 14 +- .../test_spec_engine_imputation_semantics.py | 6 +- .../shared/test_spec_engine_legacy_adapter.py | 6 +- .../shared/test_spec_engine_loader.py | 2 +- .../engine_free/us/test_puf_qrf_chain.py | 18 +- .../us/test_us_buildj_base_script.py | 2 + .../us/test_us_bundle_core_contracts.py | 2 +- .../engine_free/us/test_us_l0_refit_export.py | 18 +- .../us/test_us_late_producer_dag.py | 28 +- .../engine_free/us/test_us_multispine_pool.py | 24 +- .../us/test_us_multispine_pool_h5_io.py | 2 +- .../us/test_us_puf_e19200_decomposition.py | 382 ++++++++++++-- .../engine_free/us/test_us_puf_support.py | 83 +++- .../us/test_us_release_head_to_head_scorer.py | 6 +- .../engine_free/us/test_us_spec_bundle.py | 109 +++- .../engine_free/us/test_us_stacked_spine.py | 72 ++- .../us/test_us_multispine_pool_tool.py | 4 +- .../concept-coverage/policyengine-us.json | 10 +- .../spec_engine_inventory_coverage.py | 18 +- .../microcosm_build/us_multispine_pool.py | 2 +- .../us_multispine_pool_tool.py | 2 +- .../microcosm_build/us_puf_support.py | 1 + .../microcosm_build/us_spec_bundle.py | 2 +- ...uild_us_release_input_coverage_manifest.py | 21 + tools/generate_us_bundle_from_constants.py | 36 +- tools/refresh_concept_coverage.py | 40 +- tools/spec_engine_coverage.py | 2 +- tools/us_bundle_generation/contracts.py | 17 +- tools/us_bundle_generation/core.py | 26 +- tools/us_bundle_generation/imputation.py | 21 +- .../pending_engine_inputs.py | 39 ++ 71 files changed, 1916 insertions(+), 605 deletions(-) create mode 100644 changelog.d/us-e19200-points-premiums-investment.changed.md create mode 100644 tools/us_bundle_generation/pending_engine_inputs.py diff --git a/changelog.d/us-e19200-points-premiums-investment.changed.md b/changelog.d/us-e19200-points-premiums-investment.changed.md new file mode 100644 index 0000000000..78e102c547 --- /dev/null +++ b/changelog.d/us-e19200-points-premiums-investment.changed.md @@ -0,0 +1,7 @@ +Split the PUF E19200 non-mortgage residual into Person-level deductible points, +qualified mortgage-insurance premiums and investment interest using published +SOI AGI-band shares. Preserve the existing mortgage component and exactly +conserve each record through nested complement splits. Extend donor, QRF, +quarantine, export and declarative contracts; invalidate old QRF checkpoints. +The two new PolicyEngine-US inputs require a follow-up engine release and pin +bump before live engine contracts and dataset builds can pass. diff --git a/docs/concept-coverage/policyengine-us.md b/docs/concept-coverage/policyengine-us.md index 9c029adf6f..44c8c26fbf 100644 --- a/docs/concept-coverage/policyengine-us.md +++ b/docs/concept-coverage/policyengine-us.md @@ -2,10 +2,10 @@ Engine version `2.2.1`; concept schema `d824f5e97d58`. Generated by `tools/refresh_concept_coverage.py`; do not edit by hand. -- Engine inputs: 926 +- Engine inputs: 928 - Fed by a concept: 55 - Structural: 12 -- Uncovered: 859 +- Uncovered: 861 - Mapped but missing from the engine: 0 ## Inputs each concept feeds @@ -53,9 +53,9 @@ Engine version `2.2.1`; concept schema `d824f5e97d58`. Generated by `tools/refre `ak_ssp_excluded_institutional_setting`, `auto_loan_balance`, `auto_loan_interest`, `average_home_energy_use_in_state`, `bedrooms`, `block_geoid`, `cbo_corporate_income_tax`, `cbo_excise_tax`, `cbsa_code`, `congressional_district_geoid`, `county_fips`, `ct_tfa_region`, `current_home_energy_use`, `de_ssp_living_arrangement`, `household_business_assets_debt`, `household_business_assets_equity`, `household_business_assets_value`, `household_count`, `household_income_ami_ratio`, `household_other_real_estate_debt`, `household_other_real_estate_equity`, `household_other_real_estate_value`, `household_rental_property_debt`, `household_rental_property_equity`, `household_rental_property_value`, `household_vehicles_debt`, `household_vehicles_equity`, `household_vehicles_owned`, `household_vehicles_value`, `in_denver`, `in_wilmington`, `in_yonkers`, `is_homeless`, `is_non_english_speaking_home`, `is_on_tribal_land`, `is_rural`, `is_shared_living`, `is_sro`, `living_arrangements_allow_for_food_preparation`, `ma_state_living_arrangement`, `net_worth`, `ny_drive_clean_purchased_qualifying_vehicle`, `ny_drive_clean_vehicle_cost`, `ny_drive_clean_vehicle_electric_range`, `parent_has_less_than_high_school_education`, `place_fips`, `puma`, `qualified_passenger_vehicle_loan_interest`, `sldl`, `sldu`, `state_fips`, `tenant_pays_utilities`, `tract_geoid`, `vtd`, `zcta`, `zip_code` -### person (562) +### person (564) -`able_contributions_person`, `adopted_this_year`, `after_school_expenses`, `ak_ccap_care_schedule`, `ak_ccap_charged_rate`, `ak_ccap_provider_type`, `ak_resides_in_assisted_living_home`, `al_ccsp_provider_type`, `al_ccsp_quality_tier`, `al_level_b_care`, `al_meets_snf_criteria`, `al_receives_elderly_disabled_medicaid_waiver`, `al_ssp_care_setting`, `al_ssp_grandfathered`, `al_ssp_living_arrangement`, `al_ui_base_period_wages`, `al_ui_high_quarter_wages`, `al_ui_quarters_with_wages`, `al_ui_second_high_quarter_wages`, `al_ui_weekly_earnings`, `alimony_expense`, `alimony_income`, `ambulance_expense`, `american_opportunity_credit_claimed_prior_years`, `amt_foreign_tax_credit`, `ar_sra_care_type`, `attends_eligible_educational_institution_for_american_opportunity_credit`, `attends_eligible_educational_institution_for_lifetime_learning_credit`, `az_530a_distribution`, `az_ccap_provider_type`, `az_ccap_quality_enhanced_provider`, `bank_account_assets`, `bond_assets`, `business_is_qualified`, `business_is_sstb`, `ca_calworks_child_care_factor_category`, `ca_calworks_child_care_provider_category`, `ca_calworks_child_care_time_category`, `ca_calworks_child_care_welfare_to_work`, `ca_cvrp_vehicle_rebate_amount`, `ca_in_home_supportive_services`, `ca_in_medical_care_facility`, `ca_state_disability_insurance`, `care_and_support_costs`, `care_and_support_payments_from_tax_filer`, `casualty_loss`, `charitable_cash_donations`, `charitable_non_cash_donations`, `charitable_non_cash_donations_non_50_pct_orgs`, `child_support_expense`, `child_support_received`, `childcare_attending_days_per_month`, `childcare_days_per_week`, `childcare_hours_per_day`, `childcare_provider_type_group`, `claimed_as_dependent_on_another_return`, `claimed_ma_covid_19_essential_employee_premium_pay_program_2020`, `co_care_worker_credit_eligible_care_worker`, `co_denver_employee_occupational_privilege_tax_months`, `co_glendale_employee_occupational_privilege_tax_months`, `co_greenwood_village_employee_occupational_privilege_tax_months`, `co_quality_rating_of_child_care_facility`, `co_sheridan_employee_occupational_privilege_tax_months`, `cost_of_attending_college`, `count_529_contribution_beneficiaries`, `cps_race`, `csrs_retirement_pay`, `ct_c4k_provider_accredited`, `ct_c4k_provider_type`, `ct_ssp_has_therapeutic_diet`, `ct_ssp_lives_with_others`, `ct_ssp_resides_in_boarding_home`, `current_pregnancy_month`, `dc_ccsp_childcare_provider_category`, `dc_ccsp_schedule_type`, `dc_ossp_facility_size`, `dc_ossp_in_adult_foster_care`, `de_additions`, `de_poc_provider_type`, `de_wilmington_nonresident_earnings`, `debt_relief`, `dependent_care_employer_benefits`, `detailed_industry_recode`, `detailed_occupation_recode`, `disability_benefits`, `divorce_year`, `e00700`, `e01100`, `e01200`, `e02000`, `e03220`, `e03230`, `e03240`, `e03270`, `e03290`, `e03300`, `e03400`, `e03500`, `e07240`, `e07260`, `e07300`, `e09700`, `e09800`, `e09900`, `e11200`, `e18500`, `e19200`, `e19800`, `e20100`, `e20400`, `e24515`, `e24518`, `e26270`, `e27200`, `e32800`, `e58990`, `e62900`, `e87521`, `e87530`, `early_withdrawal_penalty`, `educational_assistance`, `educator_expense`, `employer_contribution_to_health_insurance_premiums_category`, `employer_headcount`, `employer_ny_mctmt_zone_1_quarterly_payroll_expense`, `employer_ny_mctmt_zone_2_quarterly_payroll_expense`, `employer_quarterly_payroll_expense_override`, `employer_sponsored_insurance_premiums`, `employer_state_unemployment_tax_rate_override`, `employer_total_co_denver_occupational_privilege_tax_employee_months`, `employer_total_co_denver_occupational_privilege_tax_owner_months`, `employer_total_co_glendale_occupational_privilege_tax_employee_months`, `employer_total_co_glendale_occupational_privilege_tax_owner_months`, `employer_total_co_greenwood_village_occupational_privilege_tax_employee_months`, `employer_total_co_greenwood_village_occupational_privilege_tax_owner_months`, `employer_total_co_sheridan_occupational_privilege_tax_employee_months`, `employer_total_mo_st_louis_payroll_expense`, `employer_total_or_lane_transit_district_taxable_wages`, `employer_total_or_trimet_taxable_wages`, `employer_total_payroll_tax_gross_wages`, `employer_total_taxable_earnings_for_federal_unemployment_tax`, `employer_total_taxable_earnings_for_social_security`, `employer_total_taxable_earnings_for_state_unemployment_tax`, `employer_total_wa_seattle_payroll_expense_current_year_total`, `employer_total_wa_seattle_payroll_expense_lower_band_taxable_payroll`, `employer_total_wa_seattle_payroll_expense_prior_year_total`, `employer_total_wa_seattle_payroll_expense_upper_band_taxable_payroll`, `employer_total_wa_seattle_social_housing_excess_compensation`, `er_visit_expense`, `estate_income`, `estate_income_would_be_qualified`, `excess_withheld_payroll_tax`, `farm_income`, `farm_operations_income_would_be_qualified`, `farm_rent_income`, `farm_rent_income_would_be_qualified`, `financial_assistance`, `fl_oss_community_care_type`, `fl_oss_program_track`, `fl_sr_provider_type`, `four_year_college_student`, `fsla_overtime_premium`, `ga_caps_care_type`, `ga_caps_provider_type`, `ga_caps_quality_rating`, `ga_ssp_in_nursing_home_or_institutionalized_hospice`, `gambling_losses`, `gambling_winnings`, `general_assistance`, `general_business_credit`, `gi_cash_assistance`, `has_american_opportunity_credit_1098_t_or_exception`, `has_american_opportunity_credit_institution_ein`, `has_applied_for_unemployment_compensation`, `has_bcc_qualifying_coverage`, `has_champva_health_coverage_at_interview`, `has_chip_health_coverage_at_interview`, `has_developmental_delay`, `has_emergency_medical_condition`, `has_esi`, `has_felony_drug_conviction`, `has_health_coverage_other_than_chip`, `has_indian_health_service_coverage_at_interview`, `has_individualized_education_program`, `has_lifetime_learning_credit_1098_t_or_exception`, `has_long_distance_medical_travel_for_medicaid_ce`, `has_marketplace_health_coverage_at_interview`, `has_medicaid_health_coverage_at_interview`, `has_never_worked`, `has_no_prior_formal_early_learning`, `has_non_marketplace_direct_purchase_health_coverage_at_interview`, `has_other_means_tested_health_coverage_at_interview`, `has_tricare_health_coverage_at_interview`, `has_va_health_coverage_at_interview`, `health_insurance_premiums`, `health_insurance_premiums_without_medicare_part_b`, `health_savings_account_payroll_contributions`, `heating_expense_person`, `hi_ccap_provider_category`, `hi_oss_care_facility_type`, `home_equity`, `home_is_on_agricultural_land`, `hourly_wage`, `hours_worked_last_week`, `ia_cca_provider_type`, `ia_cca_quality_rating`, `id_aabd_living_arrangement`, `id_iccp_provider_type`, `il_aabd_is_bedfast`, `il_ccap_base_day_type`, `illicit_income`, `imaging_expense`, `immigration_status_str`, `in_ccdf_provider_type`, `in_ccdf_ptq_level`, `in_out_of_home_care_facility`, `in_resides_in_licensed_residential_facility`, `in_resides_in_unlicensed_residential_facility`, `inpatient_expense`, `investment_expenses`, `investment_in_529_plan_indv`, `investment_income_elected_form_4952`, `investment_interest_expense`, `is_american_indian_or_alaska_native_for_medicaid_ce`, `is_blind`, `is_born_outside_us`, `is_breastfeeding`, `is_complying_with_tanf_work_requirements`, `is_computer_scientist`, `is_deaf`, `is_deceased`, `is_english_proficient`, `is_enrolled_at_least_half_time_for_american_opportunity_credit`, `is_enrolled_in_ccdf`, `is_enrolled_in_head_start`, `is_executive_administrative_professional`, `is_farmer_fisher`, `is_federal_work_study_participant`, `is_fully_disabled_service_connected_veteran`, `is_grandparent_of_filer_or_spouse`, `is_hispanic`, `is_hospitalized_for_medicaid_ce`, `is_in_adult_residential_care`, `is_in_foster_care`, `is_in_foster_care_group_home`, `is_in_k12_nonpublic_school`, `is_in_medicaid_community_engagement_treatment_program`, `is_in_medicaid_facility`, `is_in_residential_care_facility`, `is_in_substance_use_treatment_program`, `is_in_work_program`, `is_incapable_of_self_care`, `is_incarcerated`, `is_medicaid_1115_mec_adult`, `is_medicaid_community_engagement_seasonal_worker`, `is_medically_frail_or_has_special_medical_needs_for_medicaid_ce`, `is_migratory_child`, `is_military`, `is_paid_hourly`, `is_parent_of_filer_or_spouse`, `is_permanently_and_totally_disabled`, `is_permanently_disabled_veteran`, `is_pregnant`, `is_pursuing_credential_for_american_opportunity_credit`, `is_related_to_head_or_spouse`, `is_runaway_child`, `is_self_employed`, `is_severely_disabled`, `is_snap_abawd_discretionary_exempt`, `is_snap_abawd_indian_exempt`, `is_snap_employment_training_student`, `is_snap_work_incentive_student`, `is_snap_work_program_participant`, `is_snap_work_registration_noncompliant`, `is_snap_workfare_participant`, `is_spm_independent_minor_role`, `is_surviving_child_of_disabled_veteran`, `is_surviving_spouse_of_disabled_veteran`, `is_tafdc_related_to_head_or_spouse`, `is_union_member_or_covered`, `is_wccc_enrolled`, `is_wic_at_nutritional_risk`, `is_wic_fully_breastfeeding`, `keogh_distributions`, `ks_ccap_provider_type`, `ky_additions`, `ky_ccap_provider_type`, `ky_service_credit_months_post_1997`, `ky_service_credit_months_pre_1998`, `ky_ssp_care_receivers`, `ky_ssp_category`, `la_ccap_provider_type`, `la_quality_rating_of_child_care_facility`, `la_receives_blind_exemption`, `lab_expense`, `life_insurance_benefits`, `long_term_capital_gains_basis`, `long_term_capital_gains_on_assets_eligible_for_vt_exclusion`, `long_term_capital_gains_on_collectibles`, `long_term_capital_gains_on_small_business_stock`, `long_term_capital_gains_years_held`, `long_term_health_insurance_premiums`, `ma_ccfa_care_provider_type`, `ma_ccfa_is_in_relatives_home_care`, `ma_ccfa_schedule_type`, `major_industry_recode`, `md_ccs_provider_type`, `md_paa_living_arrangement`, `md_paa_pending_federal_benefit`, `me_ccap_provider_type`, `me_ssp_living_arrangement`, `medicaid_claiming_tax_unit_id`, `medicaid_community_engagement_community_service_hours`, `medicaid_community_engagement_less_than_half_time_education_hours`, `medicaid_community_engagement_six_month_average_income`, `medicaid_community_engagement_work_program_hours`, `medicaid_tax_dependent_exception_non_custodial_parent`, `medicare_part_b_premiums_reported`, `medicare_quarters_of_coverage`, `meets_ssi_disability_criteria`, `mi_ccap_exempt_level`, `mi_ccap_provider_type`, `mi_ccap_star_rating`, `mi_ssp_living_arrangement`, `military_basic_pay`, `military_retirement_pay`, `military_retirement_pay_survivors`, `military_service_income`, `miscellaneous_income`, `mn_ccap_provider_type`, `mn_ccap_quality_rating`, `mn_msa_has_guardian`, `mn_msa_housing_assistance_eligible`, `mn_msa_living_arrangement`, `mn_msa_uses_representative_payee`, `mo_ccs_provider_type`, `mo_kansas_city_earnings_tax_taxable_earnings`, `mo_snc_facility_base_charge`, `mo_ssp_living_arrangement`, `mo_st_louis_earnings_tax_credit`, `mo_st_louis_earnings_tax_taxable_earnings`, `months_receiving_social_security_disability`, `ms_ccpp_provider_type`, `mt_ccap_care_time`, `mt_ccap_provider_type`, `nd_ccap_provider_qris_step`, `nd_ccap_provider_type`, `ne_aabd_alternate_living_arrangement`, `ne_child_care_subsidy_provider_type`, `ne_child_care_subsidy_quality_tier`, `ne_school_readiness_credit_child_care_worker_rating`, `ne_school_readiness_credit_eligible_worker`, `never_eligible_for_social_security_benefits`, `nh_ccap_provider_type`, `nj_ccap_grow_nj_kids_rating`, `nj_ccap_provider_type`, `nj_unemployment_insurance_base_period_wages`, `nj_unemployment_insurance_base_period_weeks`, `nj_unemployment_insurance_qualifying_dependents`, `nj_unemployment_insurance_spouse_employed_when_claim_established`, `nj_unemployment_insurance_weekly_gross_wages`, `nj_unemployment_insurance_weeks_claimed`, `nj_unemployment_insurance_working_less_than_full_time`, `nm_ccap_focus_level`, `nm_ccap_non_traditional_hours`, `nm_ccap_provider_type`, `non_public_school_tuition`, `non_sch_d_capital_gains`, `nv_ccdp_provider_star_rating`, `nv_ccdp_provider_type`, `ny_ui_base_period_wages`, `ny_ui_gross_weekly_earnings`, `ny_ui_high_quarter_wages`, `ny_ui_quarters_with_wages`, `ny_ui_second_high_quarter_wages`, `ny_ui_weekly_hours_worked`, `ny_yonkers_nonresident_earnings`, `offered_aca_disqualifying_esi`, `oh_bonus_depreciation_add_back`, `oh_ccap_has_special_accommodations`, `oh_ccap_is_accredited`, `oh_ccap_non_traditional_hours`, `oh_ccap_owf_transitional`, `oh_ccap_provider_type`, `oh_ccap_quality_tier`, `oh_educator_expense_deduction_person`, `oh_federal_conformity_deductions`, `oh_has_taken_oh_lump_sum_credits`, `oh_homestead_exemption_qualifying_surviving_spouse`, `oh_other_add_backs`, `oh_section_179_expense_add_back`, `oh_uniformed_services_retirement_income_deduction`, `ok_ccs_provider_setting`, `ok_ccs_star_level`, `ok_ui_base_period_taxable_wages`, `ok_ui_base_period_total_wages`, `ok_ui_gross_weekly_earnings`, `ok_ui_high_quarter_taxable_wages`, `ok_ui_high_quarter_total_wages`, `or_erdc_provider_area`, `or_erdc_provider_type`, `other_credits`, `other_health_insurance_premiums`, `other_medical_expenses`, `outpatient_expense`, `over_the_counter_health_expenses`, `p08000`, `pa_ccw_provider_type`, `pa_philadelphia_wage_tax_reduced_rate_eligible`, `pa_philadelphia_wage_tax_resident`, `pa_philadelphia_wage_tax_taxable_wages`, `pa_uc_base_year_wages`, `pa_uc_credit_weeks`, `pa_uc_gross_weekly_earnings`, `pa_uc_highest_quarter_wages`, `partnership_income`, `partnership_s_corp_income_would_be_qualified`, `partnership_self_employment_net_earnings`, `passive_partnership_s_corp_income`, `pell_grant_countable_assets`, `pell_grant_dependent_available_income`, `pell_grant_dependent_other_allowances`, `pell_grant_head_allowances`, `pell_grant_months_in_school`, `pension_survivors`, `person_count`, `personal_property`, `pfml_benefits`, `physician_services_expense`, `pre_subsidy_care_expenses`, `pre_subsidy_transportation_expense`, `pre_tax_health_insurance_premiums`, `prescription_expense`, `previous_year_income_available`, `primary_residence_value`, `prior_year_minimum_tax_credit`, `qualified_adoption_assistance_expense`, `qualified_bdc_income`, `qualified_reit_and_ptp_income`, `qualified_tuition_expenses`, `railroad_benefits`, `receives_medicaid`, `receives_or_needs_protective_services`, `receives_ssi`, `receives_wic`, `rental_income_would_be_qualified`, `retired_from_federal_government`, `retired_from_ky_government`, `retired_on_total_disability`, `retirement_benefits_from_ss_exempt_employment`, `ri_ccap_provider_type`, `ri_ccap_star_rating`, `ri_ccap_step_rating`, `roth_401k_contributions_desired`, `roth_403b_contributions_desired`, `roth_ira_contributions_desired`, `s_corp_income`, `salt_refund_income`, `sc_ccap_provider_type`, `sc_ccap_quality_level`, `sc_ccap_registered_quality_level`, `sc_ccap_second_child_discount_rate`, `sc_tuition_credit`, `sc_tuition_credit_eligible`, `schedule_d_capital_gain_distributions`, `sd_cca_provider_type`, `self_employed_pension_contributions_desired`, `self_employment_income_last_year`, `self_employment_income_would_be_qualified`, `social_security_dependents`, `social_security_disability`, `social_security_survivors`, `ss_aime_input`, `ssi_lives_in_another_persons_household`, `ssi_lives_in_medical_treatment_facility`, `ssi_medicaid_pays_majority_of_care`, `ssi_others_pay_all_meals`, `ssi_qualifying_quarters_earnings`, `ssi_receives_food_from_others`, `ssi_receives_outside_shelter_support`, `ssi_receives_shelter_from_others_in_household`, `ssi_reported`, `ssi_shelter_support_value`, `ssn_card_type`, `sstb_self_employment_income_would_be_qualified`, `sstb_unadjusted_basis_qualified_property`, `sstb_w2_wages_from_qualified_business`, `state_or_federal_salary`, `stock_assets`, `strike_benefits`, `student_loan_interest`, `survivor_benefits`, `tax_exempt_401k_distributions`, `tax_exempt_403b_distributions`, `tax_exempt_ira_distributions`, `tax_exempt_public_pension_income`, `tax_exempt_sep_distributions`, `tax_preparation_fees`, `taxable_401k_distributions`, `taxable_403b_distributions`, `taxable_estate_value`, `taxable_federal_pension_income`, `taxable_ira_distributions`, `taxable_public_pension_income`, `taxable_roth_conversions`, `taxable_sep_distributions`, `technical_institution_student`, `tip_income`, `tn_ccap_provider_type`, `tn_ccap_qris_tier`, `total_disability_payments`, `traditional_401k_contributions_desired`, `traditional_403b_contributions_desired`, `traditional_ira_contributions_desired`, `treasury_tipped_occupation_code`, `tx_ccs_care_schedule`, `tx_ccs_provider_rating`, `tx_ccs_provider_type`, `unadjusted_basis_qualified_property`, `under_12_months_postpartum`, `under_60_days_postpartum`, `unemployment_compensation`, `unreimbursed_business_employee_expenses`, `urgent_care_expense`, `us_bonds_for_higher_ed`, `us_govt_interest_person`, `use_reported_ssi`, `ut_ccap_provider_type`, `ut_ui_base_period_wages`, `ut_ui_gross_weekly_earnings`, `ut_ui_high_quarter_wages`, `ut_ui_quarters_with_wages`, `ut_ui_weekly_hours_worked`, `ut_ui_weeks_unemployed`, `va_ccsp_is_full_day`, `va_ccsp_provider_type`, `va_educator_expenses`, `veterans_benefits`, `vt_ccfap_provider_type`, `w2_wages_from_qualified_business`, `wa_eceap_slot_type`, `wa_pfml_average_weekly_wage_override`, `wa_pfml_leave_type`, `wa_pfml_qualifying_period_hours_worked`, `wa_wccc_day_type`, `wa_wccc_provider_type`, `was_in_foster_care`, `was_recently_incarcerated_for_medicaid_ce`, `weekly_snap_work_program_hours`, `weeks_unemployed`, `wi_shares_provider_type`, `wic_breastfeeding_infant_count`, `wic_infant_feeding_category`, `workers_compensation`, `wv_ccap_non_traditional_hours`, `wv_ccap_provider_type`, `wv_ccap_quality_tier`, `wy_ccap_provider_type`, `year_deceased`, `year_of_retirement`, `years_in_military`, `years_since_us_entry` +`able_contributions_person`, `adopted_this_year`, `after_school_expenses`, `ak_ccap_care_schedule`, `ak_ccap_charged_rate`, `ak_ccap_provider_type`, `ak_resides_in_assisted_living_home`, `al_ccsp_provider_type`, `al_ccsp_quality_tier`, `al_level_b_care`, `al_meets_snf_criteria`, `al_receives_elderly_disabled_medicaid_waiver`, `al_ssp_care_setting`, `al_ssp_grandfathered`, `al_ssp_living_arrangement`, `al_ui_base_period_wages`, `al_ui_high_quarter_wages`, `al_ui_quarters_with_wages`, `al_ui_second_high_quarter_wages`, `al_ui_weekly_earnings`, `alimony_expense`, `alimony_income`, `ambulance_expense`, `american_opportunity_credit_claimed_prior_years`, `amt_foreign_tax_credit`, `ar_sra_care_type`, `attends_eligible_educational_institution_for_american_opportunity_credit`, `attends_eligible_educational_institution_for_lifetime_learning_credit`, `az_530a_distribution`, `az_ccap_provider_type`, `az_ccap_quality_enhanced_provider`, `bank_account_assets`, `bond_assets`, `business_is_qualified`, `business_is_sstb`, `ca_calworks_child_care_factor_category`, `ca_calworks_child_care_provider_category`, `ca_calworks_child_care_time_category`, `ca_calworks_child_care_welfare_to_work`, `ca_cvrp_vehicle_rebate_amount`, `ca_in_home_supportive_services`, `ca_in_medical_care_facility`, `ca_state_disability_insurance`, `care_and_support_costs`, `care_and_support_payments_from_tax_filer`, `casualty_loss`, `charitable_cash_donations`, `charitable_non_cash_donations`, `charitable_non_cash_donations_non_50_pct_orgs`, `child_support_expense`, `child_support_received`, `childcare_attending_days_per_month`, `childcare_days_per_week`, `childcare_hours_per_day`, `childcare_provider_type_group`, `claimed_as_dependent_on_another_return`, `claimed_ma_covid_19_essential_employee_premium_pay_program_2020`, `co_care_worker_credit_eligible_care_worker`, `co_denver_employee_occupational_privilege_tax_months`, `co_glendale_employee_occupational_privilege_tax_months`, `co_greenwood_village_employee_occupational_privilege_tax_months`, `co_quality_rating_of_child_care_facility`, `co_sheridan_employee_occupational_privilege_tax_months`, `cost_of_attending_college`, `count_529_contribution_beneficiaries`, `cps_race`, `csrs_retirement_pay`, `ct_c4k_provider_accredited`, `ct_c4k_provider_type`, `ct_ssp_has_therapeutic_diet`, `ct_ssp_lives_with_others`, `ct_ssp_resides_in_boarding_home`, `current_pregnancy_month`, `dc_ccsp_childcare_provider_category`, `dc_ccsp_schedule_type`, `dc_ossp_facility_size`, `dc_ossp_in_adult_foster_care`, `de_additions`, `de_poc_provider_type`, `de_wilmington_nonresident_earnings`, `debt_relief`, `dependent_care_employer_benefits`, `detailed_industry_recode`, `detailed_occupation_recode`, `disability_benefits`, `divorce_year`, `e00700`, `e01100`, `e01200`, `e02000`, `e03220`, `e03230`, `e03240`, `e03270`, `e03290`, `e03300`, `e03400`, `e03500`, `e07240`, `e07260`, `e07300`, `e09700`, `e09800`, `e09900`, `e11200`, `e18500`, `e19200`, `e19800`, `e20100`, `e20400`, `e24515`, `e24518`, `e26270`, `e27200`, `e32800`, `e58990`, `e62900`, `e87521`, `e87530`, `early_withdrawal_penalty`, `educational_assistance`, `educator_expense`, `employer_contribution_to_health_insurance_premiums_category`, `employer_headcount`, `employer_ny_mctmt_zone_1_quarterly_payroll_expense`, `employer_ny_mctmt_zone_2_quarterly_payroll_expense`, `employer_quarterly_payroll_expense_override`, `employer_sponsored_insurance_premiums`, `employer_state_unemployment_tax_rate_override`, `employer_total_co_denver_occupational_privilege_tax_employee_months`, `employer_total_co_denver_occupational_privilege_tax_owner_months`, `employer_total_co_glendale_occupational_privilege_tax_employee_months`, `employer_total_co_glendale_occupational_privilege_tax_owner_months`, `employer_total_co_greenwood_village_occupational_privilege_tax_employee_months`, `employer_total_co_greenwood_village_occupational_privilege_tax_owner_months`, `employer_total_co_sheridan_occupational_privilege_tax_employee_months`, `employer_total_mo_st_louis_payroll_expense`, `employer_total_or_lane_transit_district_taxable_wages`, `employer_total_or_trimet_taxable_wages`, `employer_total_payroll_tax_gross_wages`, `employer_total_taxable_earnings_for_federal_unemployment_tax`, `employer_total_taxable_earnings_for_social_security`, `employer_total_taxable_earnings_for_state_unemployment_tax`, `employer_total_wa_seattle_payroll_expense_current_year_total`, `employer_total_wa_seattle_payroll_expense_lower_band_taxable_payroll`, `employer_total_wa_seattle_payroll_expense_prior_year_total`, `employer_total_wa_seattle_payroll_expense_upper_band_taxable_payroll`, `employer_total_wa_seattle_social_housing_excess_compensation`, `er_visit_expense`, `estate_income`, `estate_income_would_be_qualified`, `excess_withheld_payroll_tax`, `farm_income`, `farm_operations_income_would_be_qualified`, `farm_rent_income`, `farm_rent_income_would_be_qualified`, `financial_assistance`, `fl_oss_community_care_type`, `fl_oss_program_track`, `fl_sr_provider_type`, `four_year_college_student`, `fsla_overtime_premium`, `ga_caps_care_type`, `ga_caps_provider_type`, `ga_caps_quality_rating`, `ga_ssp_in_nursing_home_or_institutionalized_hospice`, `gambling_losses`, `gambling_winnings`, `general_assistance`, `general_business_credit`, `gi_cash_assistance`, `has_american_opportunity_credit_1098_t_or_exception`, `has_american_opportunity_credit_institution_ein`, `has_applied_for_unemployment_compensation`, `has_bcc_qualifying_coverage`, `has_champva_health_coverage_at_interview`, `has_chip_health_coverage_at_interview`, `has_developmental_delay`, `has_emergency_medical_condition`, `has_esi`, `has_felony_drug_conviction`, `has_health_coverage_other_than_chip`, `has_indian_health_service_coverage_at_interview`, `has_individualized_education_program`, `has_lifetime_learning_credit_1098_t_or_exception`, `has_long_distance_medical_travel_for_medicaid_ce`, `has_marketplace_health_coverage_at_interview`, `has_medicaid_health_coverage_at_interview`, `has_never_worked`, `has_no_prior_formal_early_learning`, `has_non_marketplace_direct_purchase_health_coverage_at_interview`, `has_other_means_tested_health_coverage_at_interview`, `has_tricare_health_coverage_at_interview`, `has_va_health_coverage_at_interview`, `health_insurance_premiums`, `health_insurance_premiums_without_medicare_part_b`, `health_savings_account_payroll_contributions`, `heating_expense_person`, `hi_ccap_provider_category`, `hi_oss_care_facility_type`, `home_equity`, `home_is_on_agricultural_land`, `home_mortgage_points`, `hourly_wage`, `hours_worked_last_week`, `ia_cca_provider_type`, `ia_cca_quality_rating`, `id_aabd_living_arrangement`, `id_iccp_provider_type`, `il_aabd_is_bedfast`, `il_ccap_base_day_type`, `illicit_income`, `imaging_expense`, `immigration_status_str`, `in_ccdf_provider_type`, `in_ccdf_ptq_level`, `in_out_of_home_care_facility`, `in_resides_in_licensed_residential_facility`, `in_resides_in_unlicensed_residential_facility`, `inpatient_expense`, `investment_expenses`, `investment_in_529_plan_indv`, `investment_income_elected_form_4952`, `investment_interest_expense`, `is_american_indian_or_alaska_native_for_medicaid_ce`, `is_blind`, `is_born_outside_us`, `is_breastfeeding`, `is_complying_with_tanf_work_requirements`, `is_computer_scientist`, `is_deaf`, `is_deceased`, `is_english_proficient`, `is_enrolled_at_least_half_time_for_american_opportunity_credit`, `is_enrolled_in_ccdf`, `is_enrolled_in_head_start`, `is_executive_administrative_professional`, `is_farmer_fisher`, `is_federal_work_study_participant`, `is_fully_disabled_service_connected_veteran`, `is_grandparent_of_filer_or_spouse`, `is_hispanic`, `is_hospitalized_for_medicaid_ce`, `is_in_adult_residential_care`, `is_in_foster_care`, `is_in_foster_care_group_home`, `is_in_k12_nonpublic_school`, `is_in_medicaid_community_engagement_treatment_program`, `is_in_medicaid_facility`, `is_in_residential_care_facility`, `is_in_substance_use_treatment_program`, `is_in_work_program`, `is_incapable_of_self_care`, `is_incarcerated`, `is_medicaid_1115_mec_adult`, `is_medicaid_community_engagement_seasonal_worker`, `is_medically_frail_or_has_special_medical_needs_for_medicaid_ce`, `is_migratory_child`, `is_military`, `is_paid_hourly`, `is_parent_of_filer_or_spouse`, `is_permanently_and_totally_disabled`, `is_permanently_disabled_veteran`, `is_pregnant`, `is_pursuing_credential_for_american_opportunity_credit`, `is_related_to_head_or_spouse`, `is_runaway_child`, `is_self_employed`, `is_severely_disabled`, `is_snap_abawd_discretionary_exempt`, `is_snap_abawd_indian_exempt`, `is_snap_employment_training_student`, `is_snap_work_incentive_student`, `is_snap_work_program_participant`, `is_snap_work_registration_noncompliant`, `is_snap_workfare_participant`, `is_spm_independent_minor_role`, `is_surviving_child_of_disabled_veteran`, `is_surviving_spouse_of_disabled_veteran`, `is_tafdc_related_to_head_or_spouse`, `is_union_member_or_covered`, `is_wccc_enrolled`, `is_wic_at_nutritional_risk`, `is_wic_fully_breastfeeding`, `keogh_distributions`, `ks_ccap_provider_type`, `ky_additions`, `ky_ccap_provider_type`, `ky_service_credit_months_post_1997`, `ky_service_credit_months_pre_1998`, `ky_ssp_care_receivers`, `ky_ssp_category`, `la_ccap_provider_type`, `la_quality_rating_of_child_care_facility`, `la_receives_blind_exemption`, `lab_expense`, `life_insurance_benefits`, `long_term_capital_gains_basis`, `long_term_capital_gains_on_assets_eligible_for_vt_exclusion`, `long_term_capital_gains_on_collectibles`, `long_term_capital_gains_on_small_business_stock`, `long_term_capital_gains_years_held`, `long_term_health_insurance_premiums`, `ma_ccfa_care_provider_type`, `ma_ccfa_is_in_relatives_home_care`, `ma_ccfa_schedule_type`, `major_industry_recode`, `md_ccs_provider_type`, `md_paa_living_arrangement`, `md_paa_pending_federal_benefit`, `me_ccap_provider_type`, `me_ssp_living_arrangement`, `medicaid_claiming_tax_unit_id`, `medicaid_community_engagement_community_service_hours`, `medicaid_community_engagement_less_than_half_time_education_hours`, `medicaid_community_engagement_six_month_average_income`, `medicaid_community_engagement_work_program_hours`, `medicaid_tax_dependent_exception_non_custodial_parent`, `medicare_part_b_premiums_reported`, `medicare_quarters_of_coverage`, `meets_ssi_disability_criteria`, `mi_ccap_exempt_level`, `mi_ccap_provider_type`, `mi_ccap_star_rating`, `mi_ssp_living_arrangement`, `military_basic_pay`, `military_retirement_pay`, `military_retirement_pay_survivors`, `military_service_income`, `miscellaneous_income`, `mn_ccap_provider_type`, `mn_ccap_quality_rating`, `mn_msa_has_guardian`, `mn_msa_housing_assistance_eligible`, `mn_msa_living_arrangement`, `mn_msa_uses_representative_payee`, `mo_ccs_provider_type`, `mo_kansas_city_earnings_tax_taxable_earnings`, `mo_snc_facility_base_charge`, `mo_ssp_living_arrangement`, `mo_st_louis_earnings_tax_credit`, `mo_st_louis_earnings_tax_taxable_earnings`, `months_receiving_social_security_disability`, `mortgage_insurance_premiums`, `ms_ccpp_provider_type`, `mt_ccap_care_time`, `mt_ccap_provider_type`, `nd_ccap_provider_qris_step`, `nd_ccap_provider_type`, `ne_aabd_alternate_living_arrangement`, `ne_child_care_subsidy_provider_type`, `ne_child_care_subsidy_quality_tier`, `ne_school_readiness_credit_child_care_worker_rating`, `ne_school_readiness_credit_eligible_worker`, `never_eligible_for_social_security_benefits`, `nh_ccap_provider_type`, `nj_ccap_grow_nj_kids_rating`, `nj_ccap_provider_type`, `nj_unemployment_insurance_base_period_wages`, `nj_unemployment_insurance_base_period_weeks`, `nj_unemployment_insurance_qualifying_dependents`, `nj_unemployment_insurance_spouse_employed_when_claim_established`, `nj_unemployment_insurance_weekly_gross_wages`, `nj_unemployment_insurance_weeks_claimed`, `nj_unemployment_insurance_working_less_than_full_time`, `nm_ccap_focus_level`, `nm_ccap_non_traditional_hours`, `nm_ccap_provider_type`, `non_public_school_tuition`, `non_sch_d_capital_gains`, `nv_ccdp_provider_star_rating`, `nv_ccdp_provider_type`, `ny_ui_base_period_wages`, `ny_ui_gross_weekly_earnings`, `ny_ui_high_quarter_wages`, `ny_ui_quarters_with_wages`, `ny_ui_second_high_quarter_wages`, `ny_ui_weekly_hours_worked`, `ny_yonkers_nonresident_earnings`, `offered_aca_disqualifying_esi`, `oh_bonus_depreciation_add_back`, `oh_ccap_has_special_accommodations`, `oh_ccap_is_accredited`, `oh_ccap_non_traditional_hours`, `oh_ccap_owf_transitional`, `oh_ccap_provider_type`, `oh_ccap_quality_tier`, `oh_educator_expense_deduction_person`, `oh_federal_conformity_deductions`, `oh_has_taken_oh_lump_sum_credits`, `oh_homestead_exemption_qualifying_surviving_spouse`, `oh_other_add_backs`, `oh_section_179_expense_add_back`, `oh_uniformed_services_retirement_income_deduction`, `ok_ccs_provider_setting`, `ok_ccs_star_level`, `ok_ui_base_period_taxable_wages`, `ok_ui_base_period_total_wages`, `ok_ui_gross_weekly_earnings`, `ok_ui_high_quarter_taxable_wages`, `ok_ui_high_quarter_total_wages`, `or_erdc_provider_area`, `or_erdc_provider_type`, `other_credits`, `other_health_insurance_premiums`, `other_medical_expenses`, `outpatient_expense`, `over_the_counter_health_expenses`, `p08000`, `pa_ccw_provider_type`, `pa_philadelphia_wage_tax_reduced_rate_eligible`, `pa_philadelphia_wage_tax_resident`, `pa_philadelphia_wage_tax_taxable_wages`, `pa_uc_base_year_wages`, `pa_uc_credit_weeks`, `pa_uc_gross_weekly_earnings`, `pa_uc_highest_quarter_wages`, `partnership_income`, `partnership_s_corp_income_would_be_qualified`, `partnership_self_employment_net_earnings`, `passive_partnership_s_corp_income`, `pell_grant_countable_assets`, `pell_grant_dependent_available_income`, `pell_grant_dependent_other_allowances`, `pell_grant_head_allowances`, `pell_grant_months_in_school`, `pension_survivors`, `person_count`, `personal_property`, `pfml_benefits`, `physician_services_expense`, `pre_subsidy_care_expenses`, `pre_subsidy_transportation_expense`, `pre_tax_health_insurance_premiums`, `prescription_expense`, `previous_year_income_available`, `primary_residence_value`, `prior_year_minimum_tax_credit`, `qualified_adoption_assistance_expense`, `qualified_bdc_income`, `qualified_reit_and_ptp_income`, `qualified_tuition_expenses`, `railroad_benefits`, `receives_medicaid`, `receives_or_needs_protective_services`, `receives_ssi`, `receives_wic`, `rental_income_would_be_qualified`, `retired_from_federal_government`, `retired_from_ky_government`, `retired_on_total_disability`, `retirement_benefits_from_ss_exempt_employment`, `ri_ccap_provider_type`, `ri_ccap_star_rating`, `ri_ccap_step_rating`, `roth_401k_contributions_desired`, `roth_403b_contributions_desired`, `roth_ira_contributions_desired`, `s_corp_income`, `salt_refund_income`, `sc_ccap_provider_type`, `sc_ccap_quality_level`, `sc_ccap_registered_quality_level`, `sc_ccap_second_child_discount_rate`, `sc_tuition_credit`, `sc_tuition_credit_eligible`, `schedule_d_capital_gain_distributions`, `sd_cca_provider_type`, `self_employed_pension_contributions_desired`, `self_employment_income_last_year`, `self_employment_income_would_be_qualified`, `social_security_dependents`, `social_security_disability`, `social_security_survivors`, `ss_aime_input`, `ssi_lives_in_another_persons_household`, `ssi_lives_in_medical_treatment_facility`, `ssi_medicaid_pays_majority_of_care`, `ssi_others_pay_all_meals`, `ssi_qualifying_quarters_earnings`, `ssi_receives_food_from_others`, `ssi_receives_outside_shelter_support`, `ssi_receives_shelter_from_others_in_household`, `ssi_reported`, `ssi_shelter_support_value`, `ssn_card_type`, `sstb_self_employment_income_would_be_qualified`, `sstb_unadjusted_basis_qualified_property`, `sstb_w2_wages_from_qualified_business`, `state_or_federal_salary`, `stock_assets`, `strike_benefits`, `student_loan_interest`, `survivor_benefits`, `tax_exempt_401k_distributions`, `tax_exempt_403b_distributions`, `tax_exempt_ira_distributions`, `tax_exempt_public_pension_income`, `tax_exempt_sep_distributions`, `tax_preparation_fees`, `taxable_401k_distributions`, `taxable_403b_distributions`, `taxable_estate_value`, `taxable_federal_pension_income`, `taxable_ira_distributions`, `taxable_public_pension_income`, `taxable_roth_conversions`, `taxable_sep_distributions`, `technical_institution_student`, `tip_income`, `tn_ccap_provider_type`, `tn_ccap_qris_tier`, `total_disability_payments`, `traditional_401k_contributions_desired`, `traditional_403b_contributions_desired`, `traditional_ira_contributions_desired`, `treasury_tipped_occupation_code`, `tx_ccs_care_schedule`, `tx_ccs_provider_rating`, `tx_ccs_provider_type`, `unadjusted_basis_qualified_property`, `under_12_months_postpartum`, `under_60_days_postpartum`, `unemployment_compensation`, `unreimbursed_business_employee_expenses`, `urgent_care_expense`, `us_bonds_for_higher_ed`, `us_govt_interest_person`, `use_reported_ssi`, `ut_ccap_provider_type`, `ut_ui_base_period_wages`, `ut_ui_gross_weekly_earnings`, `ut_ui_high_quarter_wages`, `ut_ui_quarters_with_wages`, `ut_ui_weekly_hours_worked`, `ut_ui_weeks_unemployed`, `va_ccsp_is_full_day`, `va_ccsp_provider_type`, `va_educator_expenses`, `veterans_benefits`, `vt_ccfap_provider_type`, `w2_wages_from_qualified_business`, `wa_eceap_slot_type`, `wa_pfml_average_weekly_wage_override`, `wa_pfml_leave_type`, `wa_pfml_qualifying_period_hours_worked`, `wa_wccc_day_type`, `wa_wccc_provider_type`, `was_in_foster_care`, `was_recently_incarcerated_for_medicaid_ce`, `weekly_snap_work_program_hours`, `weeks_unemployed`, `wi_shares_provider_type`, `wic_breastfeeding_infant_count`, `wic_infant_feeding_category`, `workers_compensation`, `wv_ccap_non_traditional_hours`, `wv_ccap_provider_type`, `wv_ccap_quality_tier`, `wy_ccap_provider_type`, `year_deceased`, `year_of_retirement`, `years_in_military`, `years_since_us_entry` ### spm_unit (102) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index f99b29cdf6..c3d1eee24d 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "a67d71eeec9897bdfc2e00a0f8f03606d69867f6abf92598545fe116132010df", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32677, "claim_count": 49, "claims": [ { @@ -33,8 +33,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 797, - "pointer_sha256": "55c2c0cd2d652216ed35f2d2667f52fe00229eb4336507bef9758397a1e24a17", + "pointer_count": 807, + "pointer_sha256": "f35844c0d404d5fc7905d8b7ecbee1d273c0bb46a7ed656d86f1b7cb0760fb33", "rationale": null, "relative_sink_prefix": "/battery_contract", "source_prefix": "/authored/spec~1battery.yaml", @@ -108,8 +108,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 312, - "pointer_sha256": "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + "pointer_count": 314, + "pointer_sha256": "32506873791ada2618dfa6a68149bc7c403db07b45b24a2318d73283fd7ebda3", "rationale": null, "relative_sink_prefix": "/calibration_contract", "source_prefix": "/authored/spec~1calibration.yaml", @@ -136,8 +136,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 1673, - "pointer_sha256": "a983c0d68e980a31bd1ad41e7ef3c0cb3db9a7ee4d73454ae46f81eb6d1bb427", + "pointer_count": 1693, + "pointer_sha256": "8938f88fc380137cef40199d39cf5843d4bc0673e0866a42b4a6cbeacf867c61", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1catalogs.yaml/columns", @@ -274,8 +274,8 @@ ], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 24488, - "pointer_sha256": "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + "pointer_count": 24696, + "pointer_sha256": "00a65bd3961563dccb82c40f6ae00fa3b8cf4b9c0e4189af3e7f8eb5ef7277b3", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/producer_graph", @@ -334,8 +334,8 @@ ], "mode": "legacy_behavior", "pointer_class": "family_execution", - "pointer_count": 1769, - "pointer_sha256": "cd395f41c1cc425c734dc344607de043f3506b4ec2e727a29130fc4304f351e3", + "pointer_count": 1793, + "pointer_sha256": "b5d216794e2c97f3ae1ecedeb56974bf8f147c9e2f398ed935b13d7f00399d3e", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1imputation.yaml/families", @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1716, - "pointer_sha256": "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + "pointer_count": 1725, + "pointer_sha256": "1c22aa48144a38c1a757a9cf52884e77ed1a4838f0ec8a6250f89104f0ccbb66", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42457, + "consumed_field_count": 42457, "generation0_effect_counts": { - "legacy_behavior": 38498, - "no_generation0_effect": 3686 + "legacy_behavior": 38751, + "no_generation0_effect": 3706 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27945, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14010 + "legacy_behavior": 14055 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", + "pointer_inventory_sha256": "e91ed7916fe81a5dc1bff86a6f58511d6656eac977190f5a513f069af8a60701", "resolved_binding_field_count": 9780, "unused_field_count": 0 }, @@ -801,20 +801,20 @@ "early_families": 13, "early_targets": 48, "itemization_batches": 5, - "itemization_targets": 37, + "itemization_targets": 39, "late_groups": 19, - "late_targets": 70, + "late_targets": 72, "ownership_rows": 18, - "primary_effective_predictor_tuples": 65, + "primary_effective_predictor_tuples": 67, "primary_families": 1, - "primary_targets": 65, + "primary_targets": 67, "producer_authored_outputs": 92, - "producer_compiled_outputs": 227, - "producer_inputs": 2744, + "producer_compiled_outputs": 231, + "producer_inputs": 2748, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -824,11 +824,11 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 1112, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, - "typed_columns": 176, + "typed_columns": 178, "typed_entities": 8, "typed_scopes": 7 }, @@ -917,13 +917,13 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "3730d69ab29b9b7a627ed26d8c43e02b823e2a7f8e8e1a6411430d482636f9a4", + "terminal_fields": 1112 }, "failures": [], "observed": { - "sha256": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", - "terminal_fields": 934 + "sha256": "3730d69ab29b9b7a627ed26d8c43e02b823e2a7f8e8e1a6411430d482636f9a4", + "terminal_fields": 1112 }, "status": "covered" }, @@ -942,12 +942,12 @@ ], "expected": { "rows": 18, - "sha256": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" + "sha256": "d2a54260c1a4da54a247dcd723134c3b6de5cdaef021fc241d77b9e89d7dd675" }, "failures": [], "observed": { "rows": 18, - "sha256": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" + "sha256": "d2a54260c1a4da54a247dcd723134c3b6de5cdaef021fc241d77b9e89d7dd675" }, "status": "covered" }, @@ -1036,10 +1036,10 @@ 8, 8, 8, - 5 + 7 ], "batches": 5, - "targets": 37 + "targets": 39 }, "failures": [], "observed": { @@ -1048,7 +1048,7 @@ 8, 8, 8, - 5 + 7 ], "ids": [ "late/person/puf_tax_itemization__batch_1", @@ -1074,11 +1074,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" }, "failures": [], "observed": { - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "schedule_sha256": "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" }, "status": "covered" }, @@ -1096,14 +1096,14 @@ ], "expected": { "groups": 19, - "sha256": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "targets": 70 + "sha256": "d91cf5644480dd603cb04672131521f325c5053f7024eed9f831cd773e12f1b2", + "targets": 72 }, "failures": [], "observed": { "groups": 19, - "sha256": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "targets": 70 + "sha256": "d91cf5644480dd603cb04672131521f325c5053f7024eed9f831cd773e12f1b2", + "targets": 72 }, "status": "covered" }, @@ -1224,13 +1224,13 @@ "legacy_adapter.imputation.primary_qrf" ], "expected": { - "sha256": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "tuples": 65 + "sha256": "e65cc9379688863ab3172b9af8108352e003049834fc1b39445b51408e3d4ed7", + "tuples": 67 }, "failures": [], "observed": { - "sha256": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "tuples": 65 + "sha256": "e65cc9379688863ab3172b9af8108352e003049834fc1b39445b51408e3d4ed7", + "tuples": 67 }, "status": "covered" }, @@ -1248,17 +1248,17 @@ "legacy_adapter.imputation.late_producer_schedule_receipt" ], "expected": { - "edges": 71, + "edges": 70, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", - "waves": 6 + "schedule_sha256": "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116", + "waves": 5 }, "failures": [], "observed": { - "edges": 71, + "edges": 70, "nodes": 38, - "schedule_sha256": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", - "waves": 6 + "schedule_sha256": "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116", + "waves": 5 }, "status": "covered" }, @@ -1275,11 +1275,11 @@ ], "expected": { "relation": "source rows preserved exactly", - "rows": 2744 + "rows": 2748 }, "failures": [], "observed": { - "rows": 2744 + "rows": 2748 }, "status": "covered" }, @@ -1298,13 +1298,13 @@ ], "expected": { "authored_rows": 92, - "compiled_rows": 227, + "compiled_rows": 231, "relation": "typed closure exact" }, "failures": [], "observed": { "authored_rows": 92, - "compiled_rows": 227 + "compiled_rows": 231 }, "status": "covered" }, @@ -1338,12 +1338,12 @@ ], "expected": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "1d18bb017d9d609a3d851338afc13c463c3e62ead5d23ac32cc4a74bab94c4e5" }, "failures": [], "observed": { "nodes": 38, - "sha256": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e" + "sha256": "1d18bb017d9d609a3d851338afc13c463c3e62ead5d23ac32cc4a74bab94c4e5" }, "status": "covered" }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "20b4550c701dc9e2907496c01ec4f37d5ca452250b48e77682a8addf9bd47803" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "20b4550c701dc9e2907496c01ec4f37d5ca452250b48e77682a8addf9bd47803" }, "status": "covered" }, @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", - "protocol_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08" + "map_sha256": "44ccdd4707e71faa4908274816a43956a62458454e96bc2a11ae698238dbc415", + "protocol_sha256": "14141a73b2afece35ebeb416ee0399fa6f55bc86779aabe1ae755549e414f533" }, "failures": [], "observed": { - "map_sha256": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", - "protocol_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08" + "map_sha256": "44ccdd4707e71faa4908274816a43956a62458454e96bc2a11ae698238dbc415", + "protocol_sha256": "14141a73b2afece35ebeb416ee0399fa6f55bc86779aabe1ae755549e414f533" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", + "implementation_sha256": "14141a73b2afece35ebeb416ee0399fa6f55bc86779aabe1ae755549e414f533", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", + "implementation_sha256": "14141a73b2afece35ebeb416ee0399fa6f55bc86779aabe1ae755549e414f533", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", - "stages": 37 + "sha256": "7cb33a566d336fc4013dfb9aeb2bf29d4cf9726e22c39119a7cbe02256e95757", + "stages": 38 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "7cb33a566d336fc4013dfb9aeb2bf29d4cf9726e22c39119a7cbe02256e95757", "stages": 38 }, "status": "covered" @@ -1840,11 +1840,11 @@ "legacy_adapter.stacked_checkpoint_static_components" ], "expected": { - "sha256": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "sha256": "7e55d1f4b099efdb36e2127354d6ffe77678b2c42c772b150bc19265b6937aad" }, "failures": [], "observed": { - "sha256": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "sha256": "7e55d1f4b099efdb36e2127354d6ffe77678b2c42c772b150bc19265b6937aad" }, "status": "covered" }, @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "7ed03cd8e252b93ffb1f43a032e3a875cb542b972258f097dfd4ba99022af479" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "7ed03cd8e252b93ffb1f43a032e3a875cb542b972258f097dfd4ba99022af479" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "4bf59898ad7c97d8667f59b3fc1b4cd112e8b0d3c8b115ead643dadd08e64876" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323" + "spec_sha256": "a27821b016a253d7ca11d9fcbe7c8433db2852a92c2fa0209137b2a74090c5c1" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "d1df6b316e4937e809f5f020ea2d3a7bd38bccc1845f3ae6e104f7eed6a73323" + "spec_sha256": "a27821b016a253d7ca11d9fcbe7c8433db2852a92c2fa0209137b2a74090c5c1" }, "status": "pass" } diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index f3f62e66aa..fc6bb6cde3 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -243,12 +243,12 @@ by_origin_battery therefore remains unchanged. The authority versions distinguish the two contracts. The primary-QRF root - and target checkpoint schema remains version 6. The capital-gains tail + and target checkpoint schema is version 7. The capital-gains tail manifest uses schema version 2 and binds its support contract and receipt. The canonical stacked authority is version 12, the outer stacked checkpoint materializer uses version 13, and the stacked pool stage checkpoint materializer uses version 7. - The outer base identity binds primary-QRF version 6, the ACS universe and + The outer base identity binds primary-QRF version 7, the ACS universe and QBI reconciliation contracts, the tail schema and support contract, and late-producer registry schema version 17, including execution-receipt contract 4, transition authority 2, and resource-semantics receipt 2. The @@ -256,7 +256,7 @@ by_origin_battery version 1, and its checkpoint input sidecar is version 2. The companion pool manifest uses schema version 10. Older outer authority or materializer payloads are stale; primary-QRF - version 6 remains current. + version 7 remains current. The saved 10% failure checkpoint makes the ordering mechanism concrete. Before PUF it contains 385,992 clone-0 people: 342,732 ACS-origin and 43,260 @@ -300,9 +300,9 @@ by_origin_battery late-transfer banks, the primary-QRF bank, the complete 38-node DAG receipt, tail manifest and its per-status support receipt, weights audit, stack-manifest digest, fraction/seed, clone controls, and the channel-aware - producer-precedence schedule. The DAG receipt binds all 71 edges, all input - inventories, six derived waves, exact execution rows, the once-only source - finalizer, and the 19-group/70-target aggregate. Every row hashes the live + producer-precedence schedule. The DAG receipt binds all 70 edges, all input + inventories, five derived waves, exact execution rows, the once-only source + finalizer, and the 19-group/72-target aggregate. Every row hashes the live content of every declared alternative and output, the callback receipt, and the preceding row. The top receipt hashes the entry/output frames and chain terminus. Its digest is anchored in immutable Frame metadata and carried @@ -357,7 +357,7 @@ by_origin_battery boundaries. 9. A fresh `us_stacked_completeness` gate proves every declared input is observed or has exact source-by-role absence authority. The terminal - `us_by_origin_battery` then evaluates all 134 declared targets (114 person, + `us_by_origin_battery` then evaluates all 136 declared targets (116 person, 12 tax-unit, 8 SPM-unit), plus joint immigration structure, using an immutable live-digested per-column metric registry. Metric choice never dispatches from physical dtype. A digest-bound structural-absence rule may @@ -433,10 +433,10 @@ under-15 structural universe, and emits both mapped earnings columns plus `frame.@acs_pums_earnings_universe_application`. Primary PUF consumes all three outputs directly, making the universe-to-primary edge unavoidable. -The primary PUF producer has 114 external logical requirements: the following -83-input QRF/tail bundle `Q83`, plus the 31-item validation bundle `V0` below. -`Q83` consists of 17 required core rows, the optional finite -`is_full_time_college_student` tuition fallback, all 56 optional person-output +The primary PUF producer has 116 external logical requirements: the following +85-input QRF/tail bundle `Q85`, plus the 31-item validation bundle `V0` below. +`Q85` consists of 17 required core rows, the optional finite +`is_full_time_college_student` tuition fallback, all 58 optional person-output allocation bases, and all nine optional tax-unit passthroughs. Each optional row can be absent only under its own counted receipt; a present value must be finite. `V0` is the common 32-item late-transfer validation bundle `V` with @@ -444,7 +444,7 @@ only the post-PUF clone-attachment manifest removed, because primary PUF creates that manifest. The raw ACS `WAGP`/`SEMP` authority, the two ACS-scoped mapped-earnings outputs, and the application receipt add five direct dependencies, giving the executable -primary contract exactly 119 inputs. +primary contract exactly 121 inputs. ```text F(p.age) @@ -472,7 +472,7 @@ tu.@primary_qrf_checkpoint tu.@primary_puf_execution_config ``` -The 56 optional finite person allocation bases are, in canonical order: +The 58 optional finite person allocation bases are, in canonical order: ```text employment_income_before_lsr, self_employment_income_before_lsr, @@ -485,6 +485,7 @@ social_security_retirement, social_security_disability, social_security_dependents, social_security_survivors, alimony_income, alimony_expense, salt_refund_income, charitable_cash_donations, charitable_non_cash_donations, real_estate_taxes, home_mortgage_interest, +home_mortgage_points, mortgage_insurance_premiums, investment_interest_expense, investment_income_elected_form_4952, student_loan_interest, educator_expense, qualified_tuition_expenses, casualty_loss, unreimbursed_business_employee_expenses, @@ -534,17 +535,38 @@ V0 = support channel + F(clone index) on p, h, tu, s, family, marital_unit Those are 28 physical provenance/structure columns, one resolved household weight, and two metadata receipts. Primary PUF declares the same structural -surface, all six resolved-weight resources, 65 PUF/tail columns, and the clone +surface, all six resolved-weight resources, 67 PUF/tail columns, and the clone attachment manifest as outputs, so downstream dependencies are ownership edges rather than incidental observations. +The PUF E19200 decomposition preserves the existing mortgage splitter array. +Its non-mortgage residual is split by published TY2015 SOI Table 2.1 amounts +into Person-level `home_mortgage_points` (CN), `mortgage_insurance_premiums` +(CP, before any engine AGI phase-out), and `investment_interest_expense` (CR). +Nested binary complement splits conserve the four leaves exactly in real +arithmetic and reconstruct E19200 with `math.fsum`; accepted negative zero +reconstructs as positive zero, as in the original split. Donor construction +routes the mortgage array directly rather than re-multiplying its ratio. +All four leaves follow the existing raw-E19200 outlier quarantine. The new +leaves have no frozen eCPS reference share and remain explicit post-reference +release requirements. QRF checkpoint schema 7 prevents reuse of the previous +whole-residual investment target, including custom target orders. + +The concept-coverage golden anticipates the two pending inputs while keeping +the reviewed engine version pinned. Regenerate it with +`uv run --no-sync python tools/refresh_concept_coverage.py --engine policyengine-us --pending-us-e19200-leaves`. +After the PolicyEngine-US leaf release, the follow-up pin bump regenerates the +normal live-engine surface and runs `tools/repin_us_pool_engine_contracts.py`. +The current locked engine's ownership, consumer and golden checks continue to +report the two missing names. + The three primary virtual resources are semantic, not row-count assertions. The donor receipt hashes canonical typed scalar content, ordered columns, and dtypes. The checkpoint receipt binds the outer routed identity, cache mode, primary-QRF schema, manifest name, and exact target order; the physical checkpoint directory basename must independently equal that bound identity. The execution-config receipt resolves and hashes the actual predictor/output -sequences, all 65 optional allocation/passthrough reads, clone fraction/seed, +sequences, all 67 optional allocation/passthrough reads, clone fraction/seed, QRF seed/estimator count, worker module/interpreter/argv and reviewed fit environment, strict-recipient and null-preserving doctrines, the once-resolved aggregate-disaggregation spec and SOI AGI-band bytes/semantics, every tail @@ -774,7 +796,7 @@ for every primary-owned physical alternative in `V + T(E)`, including evidence for every target listed below. A target shown in both producer columns requires both scopes; that is the two-target PUF/source overlap. Thus the table is the complete per-node producer delta over `V + T(E)`, as well as the exact -70-target partition. Transfer rows abbreviate the registry's leading +72-target partition. Transfer rows abbreviate the registry's leading `transfer:`; source names in these tables abbreviate the leading `source:`. For every transfer, schema-v3 `@late_transfer_model_config` binds that node's @@ -797,10 +819,10 @@ finalizer applies the same rule to each of its sixteen source-receipt inputs. | `person/adult_care` | `is_incapable_of_self_care`, `pre_subsidy_care_expenses` | — | both from `with_us_adult_care_inputs` | | `person/model_required_boolean` | `is_pregnant` | — | from `with_us_pregnancy_inputs` | | `person/puf_tax_itemization__batch_1` | `tax_exempt_interest_income`, `long_term_capital_gains_on_collectibles`, `non_sch_d_capital_gains`, `alimony_expense`, `salt_refund_income`, `charitable_cash_donations`, `charitable_non_cash_donations`, `home_mortgage_interest` | all targets | — | -| `person/puf_tax_itemization__batch_2` | `investment_interest_expense`, `investment_income_elected_form_4952`, `student_loan_interest`, `educator_expense`, `qualified_tuition_expenses`, `casualty_loss`, `unreimbursed_business_employee_expenses`, `traditional_ira_contributions_desired` | all targets | `traditional_ira_contributions_desired` from `with_us_retirement_contribution_inputs` | -| `person/puf_tax_itemization__batch_3` | `self_employed_pension_contributions_desired`, `estate_income`, `farm_income`, `farm_rent_income`, `partnership_income`, `partnership_self_employment_net_earnings`, `estate_income_would_be_qualified`, `farm_operations_income_would_be_qualified` | all targets | `self_employed_pension_contributions_desired` from `with_us_retirement_contribution_inputs` | -| `person/puf_tax_itemization__batch_4` | `farm_rent_income_would_be_qualified`, `partnership_s_corp_income_would_be_qualified`, `rental_income_would_be_qualified`, `self_employment_income_would_be_qualified`, `sstb_self_employment_income_would_be_qualified`, `business_is_sstb`, `qualified_bdc_income`, `qualified_reit_and_ptp_income` | all targets | — | -| `person/puf_tax_itemization__batch_5` | `sstb_self_employment_income_before_lsr`, `sstb_unadjusted_basis_qualified_property`, `sstb_w2_wages_from_qualified_business`, `unadjusted_basis_qualified_property`, `w2_wages_from_qualified_business` | all targets | — | +| `person/puf_tax_itemization__batch_2` | `home_mortgage_points`, `mortgage_insurance_premiums`, `investment_interest_expense`, `investment_income_elected_form_4952`, `student_loan_interest`, `educator_expense`, `qualified_tuition_expenses`, `casualty_loss` | all targets | — | +| `person/puf_tax_itemization__batch_3` | `unreimbursed_business_employee_expenses`, `traditional_ira_contributions_desired`, `self_employed_pension_contributions_desired`, `estate_income`, `farm_income`, `farm_rent_income`, `partnership_income`, `partnership_self_employment_net_earnings` | all targets | `traditional_ira_contributions_desired`, `self_employed_pension_contributions_desired` from `with_us_retirement_contribution_inputs` | +| `person/puf_tax_itemization__batch_4` | `estate_income_would_be_qualified`, `farm_operations_income_would_be_qualified`, `farm_rent_income_would_be_qualified`, `partnership_s_corp_income_would_be_qualified`, `rental_income_would_be_qualified`, `self_employment_income_would_be_qualified`, `sstb_self_employment_income_would_be_qualified`, `business_is_sstb` | all targets | — | +| `person/puf_tax_itemization__batch_5` | `qualified_bdc_income`, `qualified_reit_and_ptp_income`, `sstb_self_employment_income_before_lsr`, `sstb_unadjusted_basis_qualified_property`, `sstb_w2_wages_from_qualified_business`, `unadjusted_basis_qualified_property`, `w2_wages_from_qualified_business` | all targets | — | | `person/source_operator_child_support` | `child_support_expense`, `child_support_received` | — | both from `with_us_child_support_inputs` | | `person/source_operator_disability_benefits` | `disability_benefits` | — | from `with_us_disability_benefits` | | `person/source_operator_education_inputs` | `attends_eligible_educational_institution_for_american_opportunity_credit`, `educational_assistance`, `has_american_opportunity_credit_1098_t_or_exception`, `has_american_opportunity_credit_institution_ein`, `is_enrolled_at_least_half_time_for_american_opportunity_credit`, `is_pursuing_credential_for_american_opportunity_credit` | — | all from `with_us_education_inputs` | @@ -822,10 +844,10 @@ common logical inventory, not the complete contract count: | `person/adult_care` | 94 | | `person/model_required_boolean` | 92 | | `person/puf_tax_itemization__batch_1` | 98 | -| `person/puf_tax_itemization__batch_2` | 100 | -| `person/puf_tax_itemization__batch_3` | 99 | +| `person/puf_tax_itemization__batch_2` | 99 | +| `person/puf_tax_itemization__batch_3` | 100 | | `person/puf_tax_itemization__batch_4` | 99 | -| `person/puf_tax_itemization__batch_5` | 96 | +| `person/puf_tax_itemization__batch_5` | 98 | | `person/source_operator_child_support` | 93 | | `person/source_operator_disability_benefits` | 92 | | `person/source_operator_education_inputs` | 97 | @@ -841,7 +863,7 @@ common logical inventory, not the complete contract count: #### Complete dependency edges -The following grouped tables enumerate all 71 unique producer-to-consumer edges. +The following grouped tables enumerate all 70 unique producer-to-consumer edges. Multiple values on one row are the input reasons carried by that edge. Bare source names carry the registry prefix `source:` and transfer paths carry `transfer:`. @@ -893,7 +915,7 @@ carries `B(E)` plus each PUF-owned target in that row which is not already in `B(E)`. The transfer target table and its contract counts therefore enumerate the complete input reasons for all 19 edges. -The remaining 19 edges are: +The remaining 18 edges are: | Producer | Consumer | Input reason | |---|---|---| @@ -907,8 +929,7 @@ The remaining 19 edges are: | `with_us_medicare_take_up_input` | `transfer:person/source_operator_medicare_take_up` | `takes_up_medicare_if_eligible` | | `with_us_pregnancy_inputs` | `with_us_wic_claim_input` | `is_pregnant` | | `with_us_pregnancy_inputs` | `transfer:person/model_required_boolean` | `is_pregnant` | -| `with_us_retirement_contribution_inputs` | `transfer:person/puf_tax_itemization__batch_2` | `traditional_ira_contributions_desired` source scope | -| `with_us_retirement_contribution_inputs` | `transfer:person/puf_tax_itemization__batch_3` | `self_employed_pension_contributions_desired` source scope | +| `with_us_retirement_contribution_inputs` | `transfer:person/puf_tax_itemization__batch_3` | `traditional_ira_contributions_desired`, `self_employed_pension_contributions_desired` source scope | | `with_us_retirement_contribution_inputs` | `transfer:person/source_operator_retirement_contributions` | three contribution outputs listed above | | `with_us_retirement_distribution_inputs` | `transfer:person/source_operator_retirement_distributions` | five distribution outputs listed above | | `with_us_weeks_unemployed` | `transfer:person/source_operator_weeks_unemployed` | `weeks_unemployed` | @@ -920,31 +941,30 @@ The remaining 19 edges are: Finally, there are 16 source-to-finalizer edges: each of the 16 source producers in the source-input table has one edge to `source_finalizer`, carried by its exact `p.@source_receipt:` resource. Thus the exhaustive count -is 1 universe-to-primary + 16 primary-to-source + 19 primary-to-transfer + 19 -cross/source-to-transfer + 16 source-to-finalizer = 71. +is 1 universe-to-primary + 16 primary-to-source + 19 primary-to-transfer + 18 +cross/source-to-transfer + 16 source-to-finalizer = 70. -The lexically canonical waves have sizes `(1, 1, 17, 14, 3, 2)`: +The lexically canonical waves have sizes `(1, 1, 18, 14, 4)`: 1. `acs_pums_earnings_universe`. 2. `primary_puf_qrf`. 3. Housing assistance; child support; childcare; disability; energy; immigration; Medicare; pregnancy; prior-year income; retirement contributions; retirement distributions; weeks unemployed; workers' - compensation; person PUF batches 1, 4, and 5; tax-unit PUF transfer. -4. Adult care; WIC; pregnancy transfer; person PUF batches 2 and 3; child + compensation; person PUF batches 1, 2, 4, and 5; tax-unit PUF transfer. +4. Adult care; education; WIC; pregnancy transfer; person PUF batch 3; child support, disability, immigration, Medicare, retirement-contribution, retirement-distribution, weeks-unemployed, workers'-compensation, and SPM-energy transfers. -5. Education; adult-care transfer; WIC transfer. -6. `source_finalizer` and education transfer. +5. `source_finalizer`; adult-care transfer; education transfer; WIC transfer. Registry schema version 17 and execution-receipt contract version 4 bind the canonical input declarations, outputs, edges, waves, exact kind-specific virtual-resource bindings, content-hashed execution-row schema, and immutable transition authority version 2. The schedule SHA-256 is -`e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f`; +`4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116`; the full payload SHA-256 is -`7be038d34f228d66c12b53558fc5f30c93f1b376f1058c5e4fd7e7563a88d67f`. +`5a33b949600aaa016b1a2a36efbe0fefd3f4072900f793d030a2e1d1b4b44073`. Reversing registry iteration produces those same bytes. The virtual-resource payload ledger is independently versioned: ACS-universe @@ -970,10 +990,10 @@ and valid. Neither receipt authorizes an upstream null. | PUF raw predictor sources | Every filing-status, count, and income component is observed in its declared source universe. Raw WAGP/SEMP authority is present and agrees with mapped leaves; a cross-grain source collision is rejected. A null on any eligible member fails before coercion. | Structure supplies status/count; ACS-native or ASEC-carried earnings supply earnings; early transfer supplies interest, dividends, and gains. | No. ACS under-15 WAGP/SEMP blanks are an exact source-universe state, not transfer starvation; all other source nulls fail. | | PUF tax-unit features | Every clone-1 recipient has a finite feature vector. Post-aggregation NaN, `+inf`, and `-inf` are counted by named predictor and rejected before fitting; none is coerced or snapped to zero. | Universe-aware person sums plus tax-unit structural inputs. | No. Eligible member values must be complete; the only special case is an all-child unit whose numeric-zero predictor is explicitly owned and counted by the named universe-zero rule. | | Primary QRF banks and chain | Donor/recipient banks are immutable; target order and RNG prefix are contiguous; all targets complete; live recipient identity, source-universe receipt, and feature digest match before finalization. | The processed full PUF donor and strict recipient checkpoint initialized above. | No. Mutation or missing receipt invalidates the bank; it cannot resume under legacy semantics. | -| Outer pool checkpoint identity and resume | Primary-QRF schema v6, primary execution config v5, portable worker identity v1, tail-manifest schema v2, late-registry schema v17/receipt contract v4, outer stacked materializer v13/authority v12, stacked pool-stage materializer v7, pool manifest schema v10, and the ACS-universe, QBI-mutation, tail-support, late-DAG, and signed virtual-resource-semantics identities must match exactly before any cached stage is discovered. The retiring legacy envelope remains manifest schema v4/materializer v3. | Fresh input pins, live stack receipt, scale controls, code identity, and all semantic contract identities. | No. An older stacked materializer or authority payload is stale; a self-consistent old receipt cannot reopen a checkpoint. Primary-QRF v6 remains current. | +| Outer pool checkpoint identity and resume | Primary-QRF schema v7, primary execution config v5, portable worker identity v1, tail-manifest schema v2, late-registry schema v17/receipt contract v4, outer stacked materializer v13/authority v12, stacked pool-stage materializer v7, pool manifest schema v10, and the ACS-universe, QBI-mutation, tail-support, late-DAG, and signed virtual-resource-semantics identities must match exactly before any cached stage is discovered. The retiring legacy envelope remains manifest schema v4/materializer v3. | Fresh input pins, live stack receipt, scale controls, code identity, and all semantic contract identities. | No. An older stacked materializer or authority payload is stale; a self-consistent old receipt cannot reopen a checkpoint. Primary-QRF v7 remains current. | | Clone-2 capital-gains tail | Each filing status requires as many eligible recipient households as selected q99.5 donors. Eligibility requires unique single-tax-unit PUF-detail lineage and half-weight capacity for the global maximum assigned donor weight. An adequate status assigns every selected donor once; a thin status skips as a whole with a named, counted `insufficient_support` receipt. | Completed clone-1 QRF output and full PUF tail donors. At 1%, `SINGLE` and `HEAD_OF_HOUSEHOLD` attach, `JOINT` and `SEPARATE` skip, and zero-requirement `SURVIVING_SPOUSE` is `not_applicable`. | No widening or partial attachment is permitted. All 22 AGI bands provide nearest-first fallback only inside a status. Universe-aware PUF recipients remain eligible, including explicitly receipted empty-universe tax units. | -| Late producer DAG | Before any callback, all declared inputs are filled on their required scopes or carry an input-specific counted absence receipt; numeric inputs are finite. The exact derived order, readiness rows, once-only source finalizer, and bounded transfer receipts must validate. | ACS earnings-universe materialization, primary PUF/tail, 16 source producers, and 19 bounded transfer groups execute in six derived waves. | No. The refusing producer names the unfilled input and its declared producing stage. A cycle fails at import with its path. | -| Late transfer completion | Every declared PUF-clone or ASEC source-producer cell is nonnull; all complementary recipients are filled; the allowed count for both unmodeled and residual rows is zero. | Forty-three PUF and 29 source targets, with two overlaps, supply the 70-target late surface. | No. A missing producer or recipient value is terminal at this boundary. | +| Late producer DAG | Before any callback, all declared inputs are filled on their required scopes or carry an input-specific counted absence receipt; numeric inputs are finite. The exact derived order, readiness rows, once-only source finalizer, and bounded transfer receipts must validate. | ACS earnings-universe materialization, primary PUF/tail, 16 source producers, and 19 bounded transfer groups execute in five derived waves. | No. The refusing producer names the unfilled input and its declared producing stage. A cycle fails at import with its path. | +| Late transfer completion | Every declared PUF-clone or ASEC source-producer cell is nonnull; all complementary recipients are filled; the allowed count for both unmodeled and residual rows is zero. | Forty-five PUF and 29 source targets, with two overlaps, supply the 72-target late surface. | No. A missing producer or recipient value is terminal at this boundary. | | Fit-weight audit | Every primary and post-PUF QRF fit receipts its resolved entity weight kind, and the collected fit records pass the weights audit before a transferred checkpoint can exist. | Calibrated household weights mapped by the frame to each modeled entity. | No. A missing, inconsistent, or manually substituted weight declaration fails before checkpoint emission. | | Tail preservation | Tail manifest, support decisions, attached descendants, IDs, weights, provenance, joint vector, and non-tail QRF cells remain exact after completion, transfer, derive, seed, and simulation. | The schema-v2 tail manifest and support receipt bound during the PUF pass and projected into both terminal gates. | A support receipt cannot authorize mutation. Any byte or identity change in an attached status, any descendant for a skipped status, or any receipt change fails. | | Schedule-D derive | Both transferred parent columns are finite for every person and align to every tax unit. Bounded late-transfer groups do not write this leaf; the whole-pool tax-unit derive is its sole canonical owner. | Completed late transfer plus tail replacements. | No. A residual would fail late transfer first and derive again by name. | @@ -981,8 +1001,8 @@ and valid. Neither receipt authorizes an upstream null. | Take-up seed | Every administratively seeded variable completes; transfer-owned take-up cannot use a default; only explicitly non-transfer-owned inputs may use receipted engine defaults. | Seed kernels, the complete transfer surface, and declared defaults. | Transfer-owned residuals fail. A declared default is a separate modeled state, not an insufficient-support receipt. | | SSI simulation projection | Every nullable engine input has a declared default on the disposable projection; the engine returns exactly one SSI value per person. | The persistent derived/seeded pool plus separately receipted ephemeral defaults. | A projection default can enable simulation but cannot cure the persistent pool; terminal evaluation returns to the original inputs plus SSI. | | Simulated checkpoint pair and resume | The persistent input-only frame and temporary evaluation frame must share exact assembly provenance; SSI exists only on the evaluation half. The live QBI receipt must authenticate the persistent frame at emission, durable write/load, and resume. | Derived/seeded persistent inputs plus the separately materialized SSI evaluation output. | No. A forged QBI receipt, altered persistent value, invalid SSI binding, or mismatched pair invalidates the simulated checkpoint and falls back only to an independently valid earlier stage. | -| Terminal completeness | All 134 registered targets exist; every positive-weight value is metric-valid; a null needs exact source/role authority, and post-PUF targets forbid absence authority. | The 48 early targets, 70 late targets, derived leaves, take-up inputs, and SSI output. | No. Only the canonical group-quarters rent rule reaches this gate as null; base WAGP/SEMP leaves are outside the 134-target terminal surface. | -| By-origin battery | All 134 clone-0 comparison surfaces are complete and valid before support is measured. | The terminal simulation frame, comparing ASEC and ACS native origins. | No. `insufficient_support` is assigned only after null and validity checks, so it cannot hide an upstream missing value. | +| Terminal completeness | All 136 registered targets exist; every positive-weight value is metric-valid; a null needs exact source/role authority, and post-PUF targets forbid absence authority. | The 48 early targets, 72 late targets, derived leaves, take-up inputs, and SSI output. | No. Only the canonical group-quarters rent rule reaches this gate as null; base WAGP/SEMP leaves are outside the 136-target terminal surface. | +| By-origin battery | All 136 clone-0 comparison surfaces are complete and valid before support is measured. | The terminal simulation frame, comparing ASEC and ACS native origins. | No. `insufficient_support` is assigned only after null and validity checks, so it cannot hide an upstream missing value. | | Manifest construction and canonical publication closure | Legacy and stacked builders reauthenticate QBI live output, canonical stacked authority, terminal-gate receipts, H5/diagnostics run IDs, and artifact digests before readiness can be asserted. | The validated persistent pool, immutable stage receipts, terminal gate snapshot, and atomically staged publication files. | No. Construction rejects forged or wrong-route receipts; publication begins with a non-ready tombstone, and only one fully authenticated run can replace it with a ready manifest. | The audit leaves no generic “receipted but null” path into a hard consumer. @@ -1142,7 +1162,7 @@ source ingestion and faithful schema harmonization -> derive -> seed take-up and other stochastic inputs -> simulate - -> completeness gate plus 134-target by-origin battery + -> completeness gate plus 136-target by-origin battery -> emit input-only pool, receipts, and terminal Logbook row ``` diff --git a/experiments/build_j_recert/buildj_base.sh b/experiments/build_j_recert/buildj_base.sh index 9081b8fd76..138fc3fc1b 100755 --- a/experiments/build_j_recert/buildj_base.sh +++ b/experiments/build_j_recert/buildj_base.sh @@ -35,6 +35,8 @@ required = { "pre_subsidy_rent", "self_employment_income_last_year", "previous_year_income_available", + "home_mortgage_points", + "mortgage_insurance_premiums", "investment_interest_expense", "salt_refund_income", "takes_up_medicare_if_eligible", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/calibration_semantics.py b/packages/microcosm-build/src/microcosm/build/spec_engine/calibration_semantics.py index 74b98196b0..de3ab2a1ca 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/calibration_semantics.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/calibration_semantics.py @@ -5,8 +5,8 @@ from collections.abc import Mapping from copy import deepcopy -from .canonical import sha256_json from .errors import SpecValidationError +from .imputation_semantics import restore_soi_e19200_agi_band_numbers CALIBRATION_SUMMARY_ALIASES = frozenset( { @@ -174,8 +174,7 @@ def resolve_calibration_tail_contracts( puf.pop("execution_binding_ref", None), _PUF_TAIL_EXECUTION_REF, location=( - "calibration/tail_contracts/puf_capital_gains_tail/" - "execution_binding_ref" + "calibration/tail_contracts/puf_capital_gains_tail/execution_binding_ref" ), ) @@ -231,10 +230,15 @@ def resolve_calibration_tail_contracts( binding.get("capital_gains_tail"), "primary_puf_qrf/binding/capital_gains_tail", ) - soi = _mapping( - execution_tail.get("soi_e19200_agi_bands"), - "primary_puf_qrf/binding/capital_gains_tail/soi_e19200_agi_bands", + soi = deepcopy( + dict( + _mapping( + execution_tail.get("soi_e19200_agi_bands"), + "primary_puf_qrf/binding/capital_gains_tail/soi_e19200_agi_bands", + ) + ) ) + restore_soi_e19200_agi_band_numbers(soi) runtime_agi_bands = dict( _mapping( soi.get("runtime_agi_bands"), @@ -258,7 +262,7 @@ def resolve_calibration_tail_contracts( "asset": soi.get("asset"), "asset_sha256": soi.get("asset_sha256"), "runtime_schema_version": runtime_agi_bands.get("schema_version"), - "runtime_sha256": sha256_json(runtime_agi_bands), + "runtime_sha256": runtime_agi_bands["sha256"], }, } ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 56f39195cf..5a397056d0 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_404 +EXPECTED_AUTHORED_FIELD_COUNT = 32_677 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_184 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_457 class FieldUsageError(AssertionError): @@ -341,8 +341,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: # wildcard broadening a source diff rather than an automatically accepted field. _PINS: dict[str, tuple[int, str]] = { "battery": ( - 797, - "55c2c0cd2d652216ed35f2d2667f52fe00229eb4336507bef9758397a1e24a17", + 807, + "f35844c0d404d5fc7905d8b7ecbee1d273c0bb46a7ed656d86f1b7cb0760fb33", ), "bundle_country": ( 1, @@ -361,12 +361,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "36564747627fb0058bb99e8e8c4397e41d8ecc472a20c49d4b6d7df1003a80ec", ), "calibration": ( - 312, - "9bad55b4945af1c4213c510942e7b9d22d104bf75b0d9804e4a183e1656c8393", + 314, + "32506873791ada2618dfa6a68149bc7c403db07b45b24a2318d73283fd7ebda3", ), "catalog_columns": ( - 1_673, - "a983c0d68e980a31bd1ad41e7ef3c0cb3db9a7ee4d73454ae46f81eb6d1bb427", + 1_693, + "8938f88fc380137cef40199d39cf5843d4bc0673e0866a42b4a6cbeacf867c61", ), "catalog_metadata_waivers": ( 5, @@ -401,8 +401,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "bdf40812604e7cd35d68093fe14b7cd1371cb8ab8658fd6002b254610b062781", ), "imputation_family_execution": ( - 1_769, - "cd395f41c1cc425c734dc344607de043f3506b4ec2e727a29130fc4304f351e3", + 1_793, + "b5d216794e2c97f3ae1ecedeb56974bf8f147c9e2f398ed935b13d7f00399d3e", ), "imputation_gap_fill_schedule": ( 5, @@ -421,8 +421,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "e1dd7dc5123ab0f39d08ea4939d98dd09a6fdb8e7449a7ca3125fb1ddbd5b4e9", ), "imputation_producer_graph": ( - 24_488, - "6382a3ed07016414c31462b8bedb24590d92b792405a89c927d85fa1bd2a8347", + 24_696, + "00a65bd3961563dccb82c40f6ae00fa3b8cf4b9c0e4189af3e7f8eb5ef7277b3", ), "imputation_transfer_execution": ( 97, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_716, - "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + 1_725, + "1c22aa48144a38c1a757a9cf52884e77ed1a4838f0ec8a6250f89104f0ccbb66", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/imputation_semantics.py b/packages/microcosm-build/src/microcosm/build/spec_engine/imputation_semantics.py index 0763a37ba4..7bba8ab0de 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/imputation_semantics.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/imputation_semantics.py @@ -78,8 +78,37 @@ def _legacy_float(value: object, *, location: str) -> float: return float(value) +def restore_soi_e19200_agi_band_numbers(soi: Mapping[str, object]) -> None: + """Restore runtime float identity, including exact zero residual shares.""" + + runtime = _mapping_like(soi["runtime_agi_bands"], "runtime SOI AGI bands") + for label, rows_value in ( + ("parsed", soi["agi_bands"]), + ("runtime", runtime["agi_bands"]), + ): + for index, row_value in enumerate( + _array_like(rows_value, f"{label} AGI bands") + ): + row = _mapping_like(row_value, f"{label} AGI band {index}") + for key in ( + "lower_bound", + "upper_bound", + "deductible_points_residual_share", + "qualified_mortgage_insurance_premiums_residual_share", + "investment_interest_residual_share", + "deductible_points_within_points_and_premiums_share", + ): + if row.get(key) is not None: + row[key] = _legacy_float( # type: ignore[index] + row[key], location=f"{label} AGI band {index}/{key}" + ) + runtime["sha256"] = _canonical_sha256( # type: ignore[index] + {key: value for key, value in runtime.items() if key != "sha256"} + ) + + def _restore_primary_tail_legacy_numbers(tail: dict[str, object]) -> None: - """Re-inflate only the reviewed float-valued capital-gains fields.""" + """Re-inflate the reviewed float-valued capital-gains and SOI fields.""" concentration = _mapping_like( tail["concentration_gate"], "capital-gains concentration gate" @@ -99,24 +128,8 @@ def _restore_primary_tail_legacy_numbers(tail: dict[str, object]) -> None: bucket[key] = _legacy_float( # type: ignore[index] bucket[key], location=f"capital-gains bucket {index}/{key}" ) - - soi = _mapping_like(tail["soi_e19200_agi_bands"], "SOI E19200 AGI bands") - runtime = _mapping_like(soi["runtime_agi_bands"], "runtime SOI AGI bands") - for label, rows_value in ( - ("parsed", soi["agi_bands"]), - ("runtime", runtime["agi_bands"]), - ): - for index, row_value in enumerate( - _array_like(rows_value, f"{label} AGI bands") - ): - row = _mapping_like(row_value, f"{label} AGI band {index}") - for key in ("lower_bound", "upper_bound"): - if row.get(key) is not None: - row[key] = _legacy_float( # type: ignore[index] - row[key], location=f"{label} AGI band {index}/{key}" - ) - runtime["sha256"] = _canonical_sha256( # type: ignore[index] - {key: value for key, value in runtime.items() if key != "sha256"} + restore_soi_e19200_agi_band_numbers( + _mapping_like(tail["soi_e19200_agi_bands"], "SOI E19200 AGI bands") ) @@ -1843,4 +1856,5 @@ def project_imputation_legacy_payloads( __all__ = [ "derive_primary_effective_predictor_tuples", "project_imputation_legacy_payloads", + "restore_soi_e19200_agi_band_numbers", ] diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 3fddff951a..ad3b7a88c7 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -211,13 +211,14 @@ "source:with_us_weeks_unemployed", "source:with_us_workers_compensation", "transfer:person/puf_tax_itemization__batch_1", + "transfer:person/puf_tax_itemization__batch_2", "transfer:person/puf_tax_itemization__batch_4", "transfer:person/puf_tax_itemization__batch_5", "transfer:tax_unit/puf_tax_itemization", "source:with_us_adult_care_inputs", + "source:with_us_education_inputs", "source:with_us_wic_claim_input", "transfer:person/model_required_boolean", - "transfer:person/puf_tax_itemization__batch_2", "transfer:person/puf_tax_itemization__batch_3", "transfer:person/source_operator_child_support", "transfer:person/source_operator_disability_benefits", @@ -228,11 +229,10 @@ "transfer:person/source_operator_weeks_unemployed", "transfer:person/source_operator_workers_compensation", "transfer:spm_unit/source_operator_energy_subsidy", - "source:with_us_education_inputs", - "transfer:person/adult_care", - "transfer:person/source_operator_wic_claim", "source_finalizer", + "transfer:person/adult_care", "transfer:person/source_operator_education_inputs", + "transfer:person/source_operator_wic_claim", ) EXPECTED_SEED_STREAMS = ( @@ -348,22 +348,22 @@ EXPECTED_HASHES = { "acs_group_predictors": "a927bb7ecf3e84f54c93583ab79318654514ac546aefafba67da5285615fbd60", "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", - "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", + "authority": "7e55d1f4b099efdb36e2127354d6ffe77678b2c42c772b150bc19265b6937aad", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "7ed03cd8e252b93ffb1f43a032e3a875cb542b972258f097dfd4ba99022af479", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", - "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", + "graph_nodes": "1d18bb017d9d609a3d851338afc13c463c3e62ead5d23ac32cc4a74bab94c4e5", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", - "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", - "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", - "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", - "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "seed_map": "0fa116d9f5dae4ba0378f190d6c822b92f3dcf1e3fe80c9880bcaf87431c283c", - "seed_protocol": "b4afa376b62fbc073fd10d74e40921f5ab60c1efe366e2c8504872091ea69d08", - "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "late_families": "d91cf5644480dd603cb04672131521f325c5053f7024eed9f831cd773e12f1b2", + "late_resource_semantics": "20b4550c701dc9e2907496c01ec4f37d5ca452250b48e77682a8addf9bd47803", + "late_schedule": "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116", + "ownership": "d2a54260c1a4da54a247dcd723134c3b6de5cdaef021fc241d77b9e89d7dd675", + "primary_tuples": "e65cc9379688863ab3172b9af8108352e003049834fc1b39445b51408e3d4ed7", + "seed_map": "44ccdd4707e71faa4908274816a43956a62458454e96bc2a11ae698238dbc415", + "seed_protocol": "14141a73b2afece35ebeb416ee0399fa6f55bc86779aabe1ae755549e414f533", + "source_manifest": "7cb33a566d336fc4013dfb9aeb2bf29d4cf9726e22c39119a7cbe02256e95757", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", - "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", + "tail": "3730d69ab29b9b7a627ed26d8c43e02b823e2a7f8e8e1a6411430d482636f9a4", } INVENTORY_REPORT_SCHEMA_VERSION = 1 @@ -420,20 +420,20 @@ "early_families": 13, "early_targets": 48, "itemization_batches": 5, - "itemization_targets": 37, + "itemization_targets": 39, "late_groups": 19, - "late_targets": 70, + "late_targets": 72, "ownership_rows": 18, - "primary_effective_predictor_tuples": 65, + "primary_effective_predictor_tuples": 67, "primary_families": 1, - "primary_targets": 65, + "primary_targets": 67, "producer_authored_outputs": 92, - "producer_compiled_outputs": 227, - "producer_inputs": 2_744, + "producer_compiled_outputs": 231, + "producer_inputs": 2_748, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -443,11 +443,11 @@ "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 1112, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, - "typed_columns": 176, + "typed_columns": 178, "typed_entities": 8, "typed_scopes": 7, } @@ -828,7 +828,7 @@ def add( "primary_predictor_tuples_exact", clauses={ "primary family is not unique": len(primary) == 1, - "effective tuple count differs": len(primary_tuples) == 65, + "effective tuple count differs": len(primary_tuples) == 67, "effective tuple digest differs": sha256_json(primary_tuples) == EXPECTED_HASHES["primary_tuples"], "legacy target order differs": _mapping( @@ -839,7 +839,7 @@ def add( homes=("/imputation/families", "/imputation/predictor_blocks/puf_tax_detail"), consumers=("compiler_ir.node_slices", "legacy_adapter.imputation.primary_qrf"), observed={"tuples": len(primary_tuples), "sha256": sha256_json(primary_tuples)}, - expected={"tuples": 65, "sha256": EXPECTED_HASHES["primary_tuples"]}, + expected={"tuples": 67, "sha256": EXPECTED_HASHES["primary_tuples"]}, ) qrf_params = _mapping( _mapping( @@ -882,7 +882,7 @@ def add( "late_split_ledger_exact", clauses={ "late group count differs": len(late) == 19, - "late target count differs": late_targets == 70, + "late target count differs": late_targets == 72, "late family digest differs": sha256_json(late) == EXPECTED_HASHES["late_families"], "legacy late schedule differs": _json_equal( @@ -898,7 +898,7 @@ def add( }, expected={ "groups": 19, - "targets": 70, + "targets": 72, "sha256": EXPECTED_HASHES["late_families"], }, ) @@ -913,8 +913,8 @@ def add( f"late/person/puf_tax_itemization__batch_{index}" for index in range(1, 6) ], - "itemization batch sizes differ": itemization_sizes == [8, 8, 8, 8, 5], - "itemization target count differs": sum(itemization_sizes) == 37, + "itemization batch sizes differ": itemization_sizes == [8, 8, 8, 8, 7], + "itemization target count differs": sum(itemization_sizes) == 39, "legacy late schedule differs": _json_equal( late_schedule, expected_late_schedule ), @@ -925,7 +925,7 @@ def add( "ids": [row["id"] for row in itemization], "batch_sizes": itemization_sizes, }, - expected={"batches": 5, "targets": 37, "batch_sizes": [8, 8, 8, 8, 5]}, + expected={"batches": 5, "targets": 39, "batch_sizes": [8, 8, 8, 8, 7]}, ) source_node_by_id = {str(row["id"]): row for row in source_nodes} @@ -961,8 +961,8 @@ def add( clauses={ "producer order differs": compiled.producer_graph.order == EXPECTED_PRODUCER_ORDER, - "edge count differs": len(compiled.producer_graph.edges) == 71, - "wave count differs": len(compiled.producer_graph.waves) == 6, + "edge count differs": len(compiled.producer_graph.edges) == 70, + "wave count differs": len(compiled.producer_graph.waves) == 5, "stage DAG differs": ( compiled.stage_dag.edges == compiled.producer_graph.edges and compiled.stage_dag.waves == compiled.producer_graph.waves @@ -992,8 +992,8 @@ def add( }, expected={ "nodes": 38, - "edges": 71, - "waves": 6, + "edges": 70, + "waves": 5, "schedule_sha256": EXPECTED_HASHES["late_schedule"], }, ) @@ -1012,7 +1012,7 @@ def add( "producer_inputs_exact", clauses={ "producer input rows differ": inputs_exact, - "input row count differs": input_count == 2744, + "input row count differs": input_count == 2748, }, homes=("/imputation/producer_graph/nodes/*/inputs",), consumers=( @@ -1020,7 +1020,7 @@ def add( "compiler_ir.node_slices", ), observed={"rows": input_count}, - expected={"rows": 2744, "relation": "source rows preserved exactly"}, + expected={"rows": 2748, "relation": "source rows preserved exactly"}, ) outputs_exact = set(expected_outputs) == set(compiled_by_id) and all( _json_equal( @@ -1038,7 +1038,7 @@ def add( "compiled outputs differ from typed closure": outputs_exact, "authored output count differs": authored_output_count == 92, "compiled output count differs": compiled.producer_graph.compiled_output_count - == 227, + == 231, }, homes=( "/imputation/producer_graph/nodes/*/outputs", @@ -1054,7 +1054,7 @@ def add( }, expected={ "authored_rows": 92, - "compiled_rows": 227, + "compiled_rows": 231, "relation": "typed closure exact", }, ) @@ -1199,7 +1199,7 @@ def add( "stages": len(_array(sources["stages"], "source stages")), "sha256": sha256_json(source_manifest), }, - expected={"stages": 37, "sha256": EXPECTED_HASHES["source_manifest"]}, + expected={"stages": 38, "sha256": EXPECTED_HASHES["source_manifest"]}, ) pipeline = _mapping(spine["pipeline_contract"], "spine pipeline contract") operator_order = tuple( @@ -1411,7 +1411,7 @@ def add( add( "capital_gains_tail_contract_exact", clauses={ - "tail terminal field count differs": tail_count == 934, + "tail terminal field count differs": tail_count == 1112, "resolved calibration tails differ": _json_equal( legacy.get("calibration_tail_contracts"), expected_legacy["calibration_tail_contracts"], @@ -1439,7 +1439,7 @@ def add( "legacy_adapter.stacked_checkpoint_static_components", ), observed={"terminal_fields": tail_count, "sha256": sha256_json(tail_bundle)}, - expected={"terminal_fields": 934, "sha256": EXPECTED_HASHES["tail"]}, + expected={"terminal_fields": 1112, "sha256": EXPECTED_HASHES["tail"]}, ) add( diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/calibration.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/calibration.schema.json index 5bc064fe0c..0880135069 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/calibration.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/calibration.schema.json @@ -888,6 +888,10 @@ "deductible_points_amount", "home_mortgage_interest_amount", "home_mortgage_share", + "deductible_points_residual_share", + "qualified_mortgage_insurance_premiums_residual_share", + "investment_interest_residual_share", + "deductible_points_within_points_and_premiums_share", "investment_interest_amount", "label", "lower_bound", @@ -902,6 +906,10 @@ "deductible_points_amount": {"type": "integer", "minimum": 0}, "home_mortgage_interest_amount": {"type": "integer", "minimum": 0}, "home_mortgage_share": {"type": "number", "minimum": 0, "maximum": 1}, + "deductible_points_residual_share": {"type": "number", "minimum": 0, "maximum": 1}, + "qualified_mortgage_insurance_premiums_residual_share": {"type": "number", "minimum": 0, "maximum": 1}, + "investment_interest_residual_share": {"type": "number", "minimum": 0, "maximum": 1}, + "deductible_points_within_points_and_premiums_share": {"type": "number", "minimum": 0, "maximum": 1}, "investment_interest_amount": {"type": "integer", "minimum": 0}, "label": {"type": "string", "minLength": 1}, "lower_bound": {"type": ["number", "null"]}, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json index be210f2657..94671af921 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/imputation.schema.json @@ -2761,6 +2761,26 @@ "home_mortgage_share": { "type": "number" }, + "deductible_points_residual_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "qualified_mortgage_insurance_premiums_residual_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "investment_interest_residual_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "deductible_points_within_points_and_premiums_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, "investment_interest_amount": { "type": "integer" }, @@ -2832,6 +2852,10 @@ "deductible_points_amount", "home_mortgage_interest_amount", "home_mortgage_share", + "deductible_points_residual_share", + "qualified_mortgage_insurance_premiums_residual_share", + "investment_interest_residual_share", + "deductible_points_within_points_and_premiums_share", "investment_interest_amount", "label", "lower_bound", @@ -2934,6 +2958,26 @@ "home_mortgage_share": { "type": "number" }, + "deductible_points_residual_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "qualified_mortgage_insurance_premiums_residual_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "investment_interest_residual_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "deductible_points_within_points_and_premiums_share": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, "investment_interest_amount": { "type": "integer" }, @@ -3005,6 +3049,10 @@ "deductible_points_amount", "home_mortgage_interest_amount", "home_mortgage_share", + "deductible_points_residual_share", + "qualified_mortgage_insurance_premiums_residual_share", + "investment_interest_residual_share", + "deductible_points_within_points_and_premiums_share", "investment_interest_amount", "label", "lower_bound", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 839249b199..b6f384cd6a 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1204,7 +1204,13 @@ "net_worth_source_column": { "type": "string" }, - "non_mortgage_output": { + "points_output": { + "type": "string" + }, + "premiums_output": { + "type": "string" + }, + "investment_output": { "type": "string" }, "notes": { @@ -1245,6 +1251,15 @@ "published_home_mortgage_column": { "type": "string" }, + "published_deductible_points_column": { + "type": "string" + }, + "published_mortgage_insurance_premiums_column": { + "type": "string" + }, + "published_investment_interest_column": { + "type": "string" + }, "published_source_year": { "type": "number" }, diff --git a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_reference.json b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_reference.json index df9241b055..61b11b51d4 100644 --- a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_reference.json +++ b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_reference.json @@ -4,6 +4,10 @@ "reviewed_consumerless_inputs_excluded": [ "has_marketplace_health_coverage" ], + "post_reference_inputs_without_nonzero_share_reference": { + "home_mortgage_points": "Separate SOI Table 2.1 CN leaf from PUF E19200; the frozen eCPS predates this input. Required in the release coverage manifest without inventing a reference share.", + "mortgage_insurance_premiums": "Separate SOI Table 2.1 CP leaf from PUF E19200; the frozen eCPS predates this input. Required in the release coverage manifest without inventing a reference share." + }, "source": { "repo_id": "policyengine/policyengine-us-data", "repo_type": "model", diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 57ecbcab6b..a29a50af62 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -166,6 +166,10 @@ "home_mortgage_interest": { "status": "required" }, + "home_mortgage_points": { + "note": "Deductible points not already carried in home_mortgage_interest (Schedule A line 8c), derived from PUF E19200 using published SOI Table 2.1 column CN within each AGI band's non-mortgage residual. No frozen eCPS nonzero-share reference exists. Required without a reviewed exclusion; pending the PolicyEngine-US leaf-input release.", + "status": "required" + }, "hourly_wage": { "status": "required" }, @@ -278,6 +282,10 @@ "miscellaneous_income": { "status": "required" }, + "mortgage_insurance_premiums": { + "note": "Qualified mortgage-insurance premiums (Schedule A line 8d), derived from PUF E19200 using published SOI Table 2.1 column CP within each AGI band's non-mortgage residual, before any engine AGI phase-out. No frozen eCPS nonzero-share reference exists. Required without a reviewed exclusion; pending the PolicyEngine-US leaf-input release.", + "status": "required" + }, "net_worth": { "status": "required" }, @@ -550,11 +558,11 @@ } }, "counts": { - "required": 167, + "required": 169, "reviewed_exclusion": 7, - "total": 174 + "total": 176 }, - "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", + "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, home_mortgage_points and mortgage_insurance_premiums from the source-backed E19200 decomposition (no frozen eCPS reference), qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", "description": "Declared full-coverage contract for a US release: every input column the reference eCPS exports must be persisted as a key with non-default signal, or carry a reviewed exclusion. Enforced as a hard release gate (microcosm.build.us_runtime.release_input_coverage) that generalizes assert_required_us_release_source_columns from 5 columns to the full eCPS input surface.", "issue": "PolicyEngine/microcosm#368", "reference": { diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 72b56c12ab..f9585a23b5 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -130,8 +130,13 @@ "published_source_year": 2015, "published_total_column": "CF", "published_home_mortgage_column": "CH", + "published_deductible_points_column": "CN", + "published_mortgage_insurance_premiums_column": "CP", + "published_investment_interest_column": "CR", "mortgage_output": "home_mortgage_interest", - "non_mortgage_output": "investment_interest_expense", + "points_output": "home_mortgage_points", + "premiums_output": "mortgage_insurance_premiums", + "investment_output": "investment_interest_expense", "allocation": "deterministic proportional allocation within each published AGI band", "outlier_screen_before_split": { "variable": "home_mortgage_interest", @@ -180,6 +185,8 @@ "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "home_mortgage_points", + "mortgage_insurance_premiums", "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", @@ -245,6 +252,8 @@ "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", + "home_mortgage_points", + "mortgage_insurance_premiums", "investment_interest_expense", "investment_income_elected_form_4952", "student_loan_interest", @@ -2415,7 +2424,7 @@ "mortgage_interest", "mortgage_principal" ], - "notes": "This structural stage consumes the mortgage-only lineage already produced by puf_tax_detail's source-backed Table 2.1 E19200 decomposition. It combines that lineage with SCF mortgage-debt hints to derive mortgage balances and person-carried mortgage inputs; it neither constructs nor overwrites investment_interest_expense." + "notes": "This structural stage consumes the mortgage-only lineage already produced by puf_tax_detail's source-backed Table 2.1 E19200 decomposition. It combines that lineage with SCF mortgage-debt hints to derive mortgage balances and person-carried mortgage inputs; it neither constructs nor overwrites home_mortgage_points, mortgage_insurance_premiums, or investment_interest_expense." }, { "stage": "acs_rent", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/battery.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/battery.yaml index ecfe34a68a..125728d641 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/battery.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/battery.yaml @@ -341,6 +341,16 @@ metric_registry: column: home_mortgage_interest clone_index: 0 metric: monetary_sign_separated +- entity: person + family: puf_tax_itemization + column: home_mortgage_points + clone_index: 0 + metric: monetary_sign_separated +- entity: person + family: puf_tax_itemization + column: mortgage_insurance_premiums + clone_index: 0 + metric: monetary_sign_separated - entity: person family: puf_tax_itemization column: investment_interest_expense diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml index 128371e097..87a689dcca 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/calibration.yaml @@ -285,6 +285,7 @@ tail_contracts: - has_never_worked - health_savings_account_ald - home_mortgage_interest + - home_mortgage_points - hourly_wage - investment_income_elected_form_4952 - investment_interest_expense @@ -299,6 +300,7 @@ tail_contracts: - long_term_capital_gains_on_collectibles - meets_ssi_disability_criteria - miscellaneous_income + - mortgage_insurance_premiums - net_worth - non_qualified_dividend_income - non_sch_d_capital_gains diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/catalogs.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/catalogs.yaml index 9d7b293817..8893221797 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/catalogs.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/catalogs.yaml @@ -537,6 +537,17 @@ columns: domain: puf_tax_itemization public_stability: internal unit_waiver: policyengine_us_unit_unavailable +- key: person.home_mortgage_points + contract: + entity: person + dtype: float64 + unit: unit_not_declared_by_engine_metadata + definition_period: year + vintage: vintage:target_2024 + nullable: false + domain: puf_tax_itemization + public_stability: internal + unit_waiver: policyengine_us_unit_unavailable - key: person.hours_worked_last_week contract: entity: person @@ -714,6 +725,17 @@ columns: domain: puf_tax_itemization public_stability: internal unit_waiver: policyengine_us_unit_unavailable +- key: person.mortgage_insurance_premiums + contract: + entity: person + dtype: float64 + unit: unit_not_declared_by_engine_metadata + definition_period: year + vintage: vintage:target_2024 + nullable: false + domain: puf_tax_itemization + public_stability: internal + unit_waiver: policyengine_us_unit_unavailable - key: person.non_qualified_dividend_income contract: entity: person diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml index 7676bdc8fe..ad8e3f8612 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/imputation.yaml @@ -202,15 +202,15 @@ chaining: reason: Declared production memory boundary preserving the canonical five-batch late PUF itemization partition. - family: late/person/puf_tax_itemization__batch_2 - after_target: traditional_ira_contributions_desired + after_target: casualty_loss reason: Declared production memory boundary preserving the canonical five-batch late PUF itemization partition. - family: late/person/puf_tax_itemization__batch_3 - after_target: farm_operations_income_would_be_qualified + after_target: partnership_self_employment_net_earnings reason: Declared production memory boundary preserving the canonical five-batch late PUF itemization partition. - family: late/person/puf_tax_itemization__batch_4 - after_target: qualified_reit_and_ptp_income + after_target: business_is_sstb reason: Declared production memory boundary preserving the canonical five-batch late PUF itemization partition. memory_policy: release_after_draw @@ -488,7 +488,7 @@ gap_fill_schedule: - name: asec_housing_to_acs activation_stage: gap_fill_stacked_spine.activation[asec_housing_to_acs] primary_checkpoint: - schema_version: 6 + schema_version: 7 target_order_digest_rule: sha256(compact_json_array) families: - id: early/asec_survey_to_acs/person/puf_tax_itemization @@ -1530,6 +1530,20 @@ families: period: year requires_concepts: [] output_coverage_scope: puf_clone + - name: home_mortgage_points + entity: person + value_kind: amount + dtype: float + period: year + requires_concepts: [] + output_coverage_scope: puf_clone + - name: mortgage_insurance_premiums + entity: person + value_kind: amount + dtype: float + period: year + requires_concepts: [] + output_coverage_scope: puf_clone - name: investment_interest_expense entity: person value_kind: amount @@ -1997,56 +2011,56 @@ families: max_targets_per_fit: 8 execution_contract: acs_transfer_default targets: - - name: investment_interest_expense + - name: home_mortgage_points entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: investment_income_elected_form_4952 + - name: mortgage_insurance_premiums entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: student_loan_interest + - name: investment_interest_expense entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: educator_expense + - name: investment_income_elected_form_4952 entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: qualified_tuition_expenses + - name: student_loan_interest entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: casualty_loss + - name: educator_expense entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: unreimbursed_business_employee_expenses + - name: qualified_tuition_expenses entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: traditional_ira_contributions_desired + - name: casualty_loss entity: person value_kind: amount dtype: float @@ -2077,59 +2091,59 @@ families: max_targets_per_fit: 8 execution_contract: acs_transfer_default targets: - - name: self_employed_pension_contributions_desired + - name: unreimbursed_business_employee_expenses entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: estate_income + - name: traditional_ira_contributions_desired entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: farm_income + - name: self_employed_pension_contributions_desired entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: farm_rent_income + - name: estate_income entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: partnership_income + - name: farm_income entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: partnership_self_employment_net_earnings + - name: farm_rent_income entity: person value_kind: amount dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: estate_income_would_be_qualified + - name: partnership_income entity: person - value_kind: flag - dtype: bool + value_kind: amount + dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: farm_operations_income_would_be_qualified + - name: partnership_self_employment_net_earnings entity: person - value_kind: flag - dtype: bool + value_kind: amount + dtype: float period: year requires_concepts: [] output_coverage_scope: whole_pool @@ -2157,59 +2171,59 @@ families: max_targets_per_fit: 8 execution_contract: acs_transfer_default targets: - - name: farm_rent_income_would_be_qualified + - name: estate_income_would_be_qualified entity: person value_kind: flag dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: partnership_s_corp_income_would_be_qualified + - name: farm_operations_income_would_be_qualified entity: person value_kind: flag dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: rental_income_would_be_qualified + - name: farm_rent_income_would_be_qualified entity: person value_kind: flag dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: self_employment_income_would_be_qualified + - name: partnership_s_corp_income_would_be_qualified entity: person value_kind: flag dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: sstb_self_employment_income_would_be_qualified + - name: rental_income_would_be_qualified entity: person value_kind: flag dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: business_is_sstb + - name: self_employment_income_would_be_qualified entity: person value_kind: flag dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: qualified_bdc_income + - name: sstb_self_employment_income_would_be_qualified entity: person - value_kind: amount - dtype: float + value_kind: flag + dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool - - name: qualified_reit_and_ptp_income + - name: business_is_sstb entity: person - value_kind: amount - dtype: float + value_kind: flag + dtype: bool period: year requires_concepts: [] output_coverage_scope: whole_pool @@ -2237,6 +2251,20 @@ families: max_targets_per_fit: 8 execution_contract: acs_transfer_default targets: + - name: qualified_bdc_income + entity: person + value_kind: amount + dtype: float + period: year + requires_concepts: [] + output_coverage_scope: whole_pool + - name: qualified_reit_and_ptp_income + entity: person + value_kind: amount + dtype: float + period: year + requires_concepts: [] + output_coverage_scope: whole_pool - name: sstb_self_employment_income_before_lsr entity: person value_kind: amount @@ -3421,6 +3449,16 @@ producer_graph: required_scope: whole_pool tolerated_absence_receipts: - optional_input:primary_puf_qrf:person_output_allocation_basis:home_mortgage_interest + - alternatives: + - - column: home_mortgage_points + entity: person + value_kind: finite_numeric + column: '@effective:person_output_allocation_basis:home_mortgage_points' + entity: person + producing_stage: post_clone_input_surface + required_scope: whole_pool + tolerated_absence_receipts: + - optional_input:primary_puf_qrf:person_output_allocation_basis:home_mortgage_points - alternatives: - - column: investment_income_elected_form_4952 entity: person @@ -3471,6 +3509,16 @@ producer_graph: required_scope: whole_pool tolerated_absence_receipts: - optional_input:primary_puf_qrf:person_output_allocation_basis:miscellaneous_income + - alternatives: + - - column: mortgage_insurance_premiums + entity: person + value_kind: finite_numeric + column: '@effective:person_output_allocation_basis:mortgage_insurance_premiums' + entity: person + producing_stage: post_clone_input_surface + required_scope: whole_pool + tolerated_absence_receipts: + - optional_input:primary_puf_qrf:person_output_allocation_basis:mortgage_insurance_premiums - alternatives: - - column: non_qualified_dividend_income entity: person @@ -4512,6 +4560,10 @@ producer_graph: lower_bound: null upper_bound: 5000.0 home_mortgage_share: 0.9537970921280867 + deductible_points_residual_share: 0.02564418040566668 + qualified_mortgage_insurance_premiums_residual_share: 0.6482054346679791 + investment_interest_residual_share: 0.3261503849263543 + deductible_points_within_points_and_premiums_share: 0.03805623663206218 - source_row: 12 total_interest_paid_amount: 1284063 home_mortgage_interest_amount: 1237451 @@ -4529,6 +4581,10 @@ producer_graph: lower_bound: 5000.0 upper_bound: 10000.0 home_mortgage_share: 0.9636996004090141 + deductible_points_residual_share: 0.018686576130098047 + qualified_mortgage_insurance_premiums_residual_share: 0.7638754800369012 + investment_interest_residual_share: 0.2174379438330008 + deductible_points_within_points_and_premiums_share: 0.02387871477135651 - source_row: 13 total_interest_paid_amount: 2148799 home_mortgage_interest_amount: 2083827 @@ -4546,6 +4602,10 @@ producer_graph: lower_bound: 10000.0 upper_bound: 15000.0 home_mortgage_share: 0.9697635749085884 + deductible_points_residual_share: 0.05422335775410946 + qualified_mortgage_insurance_premiums_residual_share: 0.8645724312011328 + investment_interest_residual_share: 0.08120421104475774 + deductible_points_within_points_and_premiums_share: 0.05901567944250871 - source_row: 14 total_interest_paid_amount: 2699978 home_mortgage_interest_amount: 2589295 @@ -4563,6 +4623,10 @@ producer_graph: lower_bound: 15000.0 upper_bound: 20000.0 home_mortgage_share: 0.9590059622708037 + deductible_points_residual_share: 0.11428132594888103 + qualified_mortgage_insurance_premiums_residual_share: 0.8468418817704616 + investment_interest_residual_share: 0.03887679228065737 + deductible_points_within_points_and_premiums_share: 0.11890392931002068 - source_row: 15 total_interest_paid_amount: 3181401 home_mortgage_interest_amount: 2992991 @@ -4580,6 +4644,10 @@ producer_graph: lower_bound: 20000.0 upper_bound: 25000.0 home_mortgage_share: 0.9407776636771032 + deductible_points_residual_share: 0.08719328694489117 + qualified_mortgage_insurance_premiums_residual_share: 0.7856843356739859 + investment_interest_residual_share: 0.12712237738112298 + deductible_points_within_points_and_premiums_share: 0.09989176567877513 - source_row: 16 total_interest_paid_amount: 3748228 home_mortgage_interest_amount: 3591380 @@ -4597,6 +4665,10 @@ producer_graph: lower_bound: 25000.0 upper_bound: 30000.0 home_mortgage_share: 0.9581540930807838 + deductible_points_residual_share: 0.08330358053657044 + qualified_mortgage_insurance_premiums_residual_share: 0.8703266857084566 + investment_interest_residual_share: 0.04636973375497297 + deductible_points_within_points_and_premiums_share: 0.08735417014875481 - source_row: 17 total_interest_paid_amount: 4214763 home_mortgage_interest_amount: 3946351 @@ -4614,6 +4686,10 @@ producer_graph: lower_bound: 30000.0 upper_bound: 35000.0 home_mortgage_share: 0.9363162294060188 + deductible_points_residual_share: 0.06072776451039637 + qualified_mortgage_insurance_premiums_residual_share: 0.9166017786156305 + investment_interest_residual_share: 0.02267045687397312 + deductible_points_within_points_and_premiums_share: 0.06213642566882429 - source_row: 18 total_interest_paid_amount: 5435778 home_mortgage_interest_amount: 5122692 @@ -4631,6 +4707,10 @@ producer_graph: lower_bound: 35000.0 upper_bound: 40000.0 home_mortgage_share: 0.9424027250561006 + deductible_points_residual_share: 0.03465501491602946 + qualified_mortgage_insurance_premiums_residual_share: 0.9232064033524335 + investment_interest_residual_share: 0.04213858173153702 + deductible_points_within_points_and_premiums_share: 0.0361795707135545 - source_row: 19 total_interest_paid_amount: 6009927 home_mortgage_interest_amount: 5651139 @@ -4648,6 +4728,10 @@ producer_graph: lower_bound: 40000.0 upper_bound: 45000.0 home_mortgage_share: 0.9403007723721103 + deductible_points_residual_share: 0.06263308694828143 + qualified_mortgage_insurance_premiums_residual_share: 0.897588548111977 + investment_interest_residual_share: 0.03977836493974157 + deductible_points_within_points_and_premiums_share: 0.06522773978567033 - source_row: 20 total_interest_paid_amount: 6888540 home_mortgage_interest_amount: 6411648 @@ -4665,6 +4749,10 @@ producer_graph: lower_bound: 45000.0 upper_bound: 50000.0 home_mortgage_share: 0.930770235782909 + deductible_points_residual_share: 0.07636320173120958 + qualified_mortgage_insurance_premiums_residual_share: 0.902040713620694 + investment_interest_residual_share: 0.021596084648096425 + deductible_points_within_points_and_premiums_share: 0.07804874912396885 - source_row: 21 total_interest_paid_amount: 6955789 home_mortgage_interest_amount: 6543517 @@ -4682,6 +4770,10 @@ producer_graph: lower_bound: 50000.0 upper_bound: 55000.0 home_mortgage_share: 0.9407296569806819 + deductible_points_residual_share: 0.05324652958854734 + qualified_mortgage_insurance_premiums_residual_share: 0.9182236926681722 + investment_interest_residual_share: 0.028529777743280513 + deductible_points_within_points_and_premiums_share: 0.05481025395184628 - source_row: 22 total_interest_paid_amount: 7424513 home_mortgage_interest_amount: 6985554 @@ -4699,6 +4791,10 @@ producer_graph: lower_bound: 55000.0 upper_bound: 60000.0 home_mortgage_share: 0.9408770649334172 + deductible_points_residual_share: 0.08112374959848186 + qualified_mortgage_insurance_premiums_residual_share: 0.8829913499894068 + investment_interest_residual_share: 0.035884900412111384 + deductible_points_within_points_and_premiums_share: 0.08414322069341953 - source_row: 23 total_interest_paid_amount: 23294604 home_mortgage_interest_amount: 21825640 @@ -4716,6 +4812,10 @@ producer_graph: lower_bound: 60000.0 upper_bound: 75000.0 home_mortgage_share: 0.9369397307633991 + deductible_points_residual_share: 0.07572746797915539 + qualified_mortgage_insurance_premiums_residual_share: 0.8665686384631356 + investment_interest_residual_share: 0.057703893557708996 + deductible_points_within_points_and_premiums_share: 0.08036483167172374 - source_row: 24 total_interest_paid_amount: 43398259 home_mortgage_interest_amount: 40897449 @@ -4733,6 +4833,10 @@ producer_graph: lower_bound: 75000.0 upper_bound: 100000.0 home_mortgage_share: 0.9423753381443251 + deductible_points_residual_share: 0.09537193764097938 + qualified_mortgage_insurance_premiums_residual_share: 0.8385202548455319 + investment_interest_residual_share: 0.06610780751348863 + deductible_points_within_points_and_premiums_share: 0.10212306988780921 - source_row: 25 total_interest_paid_amount: 104846625 home_mortgage_interest_amount: 103319155 @@ -4750,6 +4854,10 @@ producer_graph: lower_bound: 100000.0 upper_bound: 200000.0 home_mortgage_share: 0.9854313860841968 + deductible_points_residual_share: 0.30561713159669257 + qualified_mortgage_insurance_premiums_residual_share: 0.23383568908063662 + investment_interest_residual_share: 0.4605471793226708 + deductible_points_within_points_and_premiums_share: 0.5665317148828031 - source_row: 26 total_interest_paid_amount: 53562301 home_mortgage_interest_amount: 51606382 @@ -4767,6 +4875,10 @@ producer_graph: lower_bound: 200000.0 upper_bound: 500000.0 home_mortgage_share: 0.9634832902343011 + deductible_points_residual_share: 0.10577630986952431 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.8942236901304756 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 27 total_interest_paid_amount: 12836968 home_mortgage_interest_amount: 11335153 @@ -4784,6 +4896,10 @@ producer_graph: lower_bound: 500000.0 upper_bound: 1000000.0 home_mortgage_share: 0.883008588944056 + deductible_points_residual_share: 0.02727364737091628 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9727263526290837 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 28 total_interest_paid_amount: 3505209 home_mortgage_interest_amount: 2559360 @@ -4801,6 +4917,10 @@ producer_graph: lower_bound: 1000000.0 upper_bound: 1500000.0 home_mortgage_share: 0.7301590290336468 + deductible_points_residual_share: 0.008392460107268707 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9916075398927313 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 29 total_interest_paid_amount: 1666807 home_mortgage_interest_amount: 1043154 @@ -4818,6 +4938,10 @@ producer_graph: lower_bound: 1500000.0 upper_bound: 2000000.0 home_mortgage_share: 0.6258397042968982 + deductible_points_residual_share: 0.0042251127231212274 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9957748872768788 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 30 total_interest_paid_amount: 3412493 home_mortgage_interest_amount: 1521570 @@ -4835,6 +4959,10 @@ producer_graph: lower_bound: 2000000.0 upper_bound: 5000000.0 home_mortgage_share: 0.44588223331154087 + deductible_points_residual_share: 0.0026981518030338605 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9973018481969661 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 31 total_interest_paid_amount: 1535736 home_mortgage_interest_amount: 360807 @@ -4852,6 +4980,10 @@ producer_graph: lower_bound: 5000000.0 upper_bound: 10000000.0 home_mortgage_share: 0.23494077107002767 + deductible_points_residual_share: 0.0011804968640658287 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9988195031359342 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 32 total_interest_paid_amount: 5178966 home_mortgage_interest_amount: 205431 @@ -4869,6 +5001,10 @@ producer_graph: lower_bound: 10000000.0 upper_bound: null home_mortgage_share: 0.0396664121757123 + deductible_points_residual_share: 0.0001280779164115664 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9998719220835884 + deductible_points_within_points_and_premiums_share: 1.0 runtime_agi_bands: schema_version: 1 agi_bands: @@ -4889,6 +5025,10 @@ producer_graph: lower_bound: null upper_bound: 5000.0 home_mortgage_share: 0.9537970921280867 + deductible_points_residual_share: 0.02564418040566668 + qualified_mortgage_insurance_premiums_residual_share: 0.6482054346679791 + investment_interest_residual_share: 0.3261503849263543 + deductible_points_within_points_and_premiums_share: 0.03805623663206218 - source_row: 12 total_interest_paid_amount: 1284063 home_mortgage_interest_amount: 1237451 @@ -4906,6 +5046,10 @@ producer_graph: lower_bound: 5000.0 upper_bound: 10000.0 home_mortgage_share: 0.9636996004090141 + deductible_points_residual_share: 0.018686576130098047 + qualified_mortgage_insurance_premiums_residual_share: 0.7638754800369012 + investment_interest_residual_share: 0.2174379438330008 + deductible_points_within_points_and_premiums_share: 0.02387871477135651 - source_row: 13 total_interest_paid_amount: 2148799 home_mortgage_interest_amount: 2083827 @@ -4923,6 +5067,10 @@ producer_graph: lower_bound: 10000.0 upper_bound: 15000.0 home_mortgage_share: 0.9697635749085884 + deductible_points_residual_share: 0.05422335775410946 + qualified_mortgage_insurance_premiums_residual_share: 0.8645724312011328 + investment_interest_residual_share: 0.08120421104475774 + deductible_points_within_points_and_premiums_share: 0.05901567944250871 - source_row: 14 total_interest_paid_amount: 2699978 home_mortgage_interest_amount: 2589295 @@ -4940,6 +5088,10 @@ producer_graph: lower_bound: 15000.0 upper_bound: 20000.0 home_mortgage_share: 0.9590059622708037 + deductible_points_residual_share: 0.11428132594888103 + qualified_mortgage_insurance_premiums_residual_share: 0.8468418817704616 + investment_interest_residual_share: 0.03887679228065737 + deductible_points_within_points_and_premiums_share: 0.11890392931002068 - source_row: 15 total_interest_paid_amount: 3181401 home_mortgage_interest_amount: 2992991 @@ -4957,6 +5109,10 @@ producer_graph: lower_bound: 20000.0 upper_bound: 25000.0 home_mortgage_share: 0.9407776636771032 + deductible_points_residual_share: 0.08719328694489117 + qualified_mortgage_insurance_premiums_residual_share: 0.7856843356739859 + investment_interest_residual_share: 0.12712237738112298 + deductible_points_within_points_and_premiums_share: 0.09989176567877513 - source_row: 16 total_interest_paid_amount: 3748228 home_mortgage_interest_amount: 3591380 @@ -4974,6 +5130,10 @@ producer_graph: lower_bound: 25000.0 upper_bound: 30000.0 home_mortgage_share: 0.9581540930807838 + deductible_points_residual_share: 0.08330358053657044 + qualified_mortgage_insurance_premiums_residual_share: 0.8703266857084566 + investment_interest_residual_share: 0.04636973375497297 + deductible_points_within_points_and_premiums_share: 0.08735417014875481 - source_row: 17 total_interest_paid_amount: 4214763 home_mortgage_interest_amount: 3946351 @@ -4991,6 +5151,10 @@ producer_graph: lower_bound: 30000.0 upper_bound: 35000.0 home_mortgage_share: 0.9363162294060188 + deductible_points_residual_share: 0.06072776451039637 + qualified_mortgage_insurance_premiums_residual_share: 0.9166017786156305 + investment_interest_residual_share: 0.02267045687397312 + deductible_points_within_points_and_premiums_share: 0.06213642566882429 - source_row: 18 total_interest_paid_amount: 5435778 home_mortgage_interest_amount: 5122692 @@ -5008,6 +5172,10 @@ producer_graph: lower_bound: 35000.0 upper_bound: 40000.0 home_mortgage_share: 0.9424027250561006 + deductible_points_residual_share: 0.03465501491602946 + qualified_mortgage_insurance_premiums_residual_share: 0.9232064033524335 + investment_interest_residual_share: 0.04213858173153702 + deductible_points_within_points_and_premiums_share: 0.0361795707135545 - source_row: 19 total_interest_paid_amount: 6009927 home_mortgage_interest_amount: 5651139 @@ -5025,6 +5193,10 @@ producer_graph: lower_bound: 40000.0 upper_bound: 45000.0 home_mortgage_share: 0.9403007723721103 + deductible_points_residual_share: 0.06263308694828143 + qualified_mortgage_insurance_premiums_residual_share: 0.897588548111977 + investment_interest_residual_share: 0.03977836493974157 + deductible_points_within_points_and_premiums_share: 0.06522773978567033 - source_row: 20 total_interest_paid_amount: 6888540 home_mortgage_interest_amount: 6411648 @@ -5042,6 +5214,10 @@ producer_graph: lower_bound: 45000.0 upper_bound: 50000.0 home_mortgage_share: 0.930770235782909 + deductible_points_residual_share: 0.07636320173120958 + qualified_mortgage_insurance_premiums_residual_share: 0.902040713620694 + investment_interest_residual_share: 0.021596084648096425 + deductible_points_within_points_and_premiums_share: 0.07804874912396885 - source_row: 21 total_interest_paid_amount: 6955789 home_mortgage_interest_amount: 6543517 @@ -5059,6 +5235,10 @@ producer_graph: lower_bound: 50000.0 upper_bound: 55000.0 home_mortgage_share: 0.9407296569806819 + deductible_points_residual_share: 0.05324652958854734 + qualified_mortgage_insurance_premiums_residual_share: 0.9182236926681722 + investment_interest_residual_share: 0.028529777743280513 + deductible_points_within_points_and_premiums_share: 0.05481025395184628 - source_row: 22 total_interest_paid_amount: 7424513 home_mortgage_interest_amount: 6985554 @@ -5076,6 +5256,10 @@ producer_graph: lower_bound: 55000.0 upper_bound: 60000.0 home_mortgage_share: 0.9408770649334172 + deductible_points_residual_share: 0.08112374959848186 + qualified_mortgage_insurance_premiums_residual_share: 0.8829913499894068 + investment_interest_residual_share: 0.035884900412111384 + deductible_points_within_points_and_premiums_share: 0.08414322069341953 - source_row: 23 total_interest_paid_amount: 23294604 home_mortgage_interest_amount: 21825640 @@ -5093,6 +5277,10 @@ producer_graph: lower_bound: 60000.0 upper_bound: 75000.0 home_mortgage_share: 0.9369397307633991 + deductible_points_residual_share: 0.07572746797915539 + qualified_mortgage_insurance_premiums_residual_share: 0.8665686384631356 + investment_interest_residual_share: 0.057703893557708996 + deductible_points_within_points_and_premiums_share: 0.08036483167172374 - source_row: 24 total_interest_paid_amount: 43398259 home_mortgage_interest_amount: 40897449 @@ -5110,6 +5298,10 @@ producer_graph: lower_bound: 75000.0 upper_bound: 100000.0 home_mortgage_share: 0.9423753381443251 + deductible_points_residual_share: 0.09537193764097938 + qualified_mortgage_insurance_premiums_residual_share: 0.8385202548455319 + investment_interest_residual_share: 0.06610780751348863 + deductible_points_within_points_and_premiums_share: 0.10212306988780921 - source_row: 25 total_interest_paid_amount: 104846625 home_mortgage_interest_amount: 103319155 @@ -5127,6 +5319,10 @@ producer_graph: lower_bound: 100000.0 upper_bound: 200000.0 home_mortgage_share: 0.9854313860841968 + deductible_points_residual_share: 0.30561713159669257 + qualified_mortgage_insurance_premiums_residual_share: 0.23383568908063662 + investment_interest_residual_share: 0.4605471793226708 + deductible_points_within_points_and_premiums_share: 0.5665317148828031 - source_row: 26 total_interest_paid_amount: 53562301 home_mortgage_interest_amount: 51606382 @@ -5144,6 +5340,10 @@ producer_graph: lower_bound: 200000.0 upper_bound: 500000.0 home_mortgage_share: 0.9634832902343011 + deductible_points_residual_share: 0.10577630986952431 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.8942236901304756 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 27 total_interest_paid_amount: 12836968 home_mortgage_interest_amount: 11335153 @@ -5161,6 +5361,10 @@ producer_graph: lower_bound: 500000.0 upper_bound: 1000000.0 home_mortgage_share: 0.883008588944056 + deductible_points_residual_share: 0.02727364737091628 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9727263526290837 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 28 total_interest_paid_amount: 3505209 home_mortgage_interest_amount: 2559360 @@ -5178,6 +5382,10 @@ producer_graph: lower_bound: 1000000.0 upper_bound: 1500000.0 home_mortgage_share: 0.7301590290336468 + deductible_points_residual_share: 0.008392460107268707 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9916075398927313 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 29 total_interest_paid_amount: 1666807 home_mortgage_interest_amount: 1043154 @@ -5195,6 +5403,10 @@ producer_graph: lower_bound: 1500000.0 upper_bound: 2000000.0 home_mortgage_share: 0.6258397042968982 + deductible_points_residual_share: 0.0042251127231212274 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9957748872768788 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 30 total_interest_paid_amount: 3412493 home_mortgage_interest_amount: 1521570 @@ -5212,6 +5424,10 @@ producer_graph: lower_bound: 2000000.0 upper_bound: 5000000.0 home_mortgage_share: 0.44588223331154087 + deductible_points_residual_share: 0.0026981518030338605 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9973018481969661 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 31 total_interest_paid_amount: 1535736 home_mortgage_interest_amount: 360807 @@ -5229,6 +5445,10 @@ producer_graph: lower_bound: 5000000.0 upper_bound: 10000000.0 home_mortgage_share: 0.23494077107002767 + deductible_points_residual_share: 0.0011804968640658287 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9988195031359342 + deductible_points_within_points_and_premiums_share: 1.0 - source_row: 32 total_interest_paid_amount: 5178966 home_mortgage_interest_amount: 205431 @@ -5246,6 +5466,10 @@ producer_graph: lower_bound: 10000000.0 upper_bound: null home_mortgage_share: 0.0396664121757123 + deductible_points_residual_share: 0.0001280779164115664 + qualified_mortgage_insurance_premiums_residual_share: 0.0 + investment_interest_residual_share: 0.9998719220835884 + deductible_points_within_points_and_premiums_share: 1.0 concentration_gate: schema_version: 2 selection_quantile: 0.995 @@ -18186,6 +18410,15 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] + - alternatives: + - - column: home_mortgage_points + entity: person + value_kind: finite_numeric + column: home_mortgage_points + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] - alternatives: - - column: investment_income_elected_form_4952 entity: person @@ -18204,6 +18437,15 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] + - alternatives: + - - column: mortgage_insurance_premiums + entity: person + value_kind: finite_numeric + column: mortgage_insurance_premiums + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] - alternatives: - - column: non_qualified_dividend_income entity: person @@ -18420,33 +18662,6 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] - - alternatives: - - - column: traditional_ira_contributions_desired - entity: person - value_kind: finite_numeric - column: traditional_ira_contributions_desired - entity: person - producing_stage: source:with_us_retirement_contribution_inputs - required_scope: asec_source - tolerated_absence_receipts: [] - - alternatives: - - - column: traditional_ira_contributions_desired - entity: person - value_kind: finite_numeric - column: traditional_ira_contributions_desired - entity: person - producing_stage: primary_puf_qrf - required_scope: puf_clone - tolerated_absence_receipts: [] - - alternatives: - - - column: unreimbursed_business_employee_expenses - entity: person - value_kind: finite_numeric - column: unreimbursed_business_employee_expenses - entity: person - producing_stage: primary_puf_qrf - required_scope: puf_clone - tolerated_absence_receipts: [] - alternatives: - - column: spm_unit_id entity: spm_unit @@ -18579,14 +18794,14 @@ producer_graph: entity: person family: puf_tax_itemization__batch_2 ordered_targets: + - home_mortgage_points + - mortgage_insurance_premiums - investment_interest_expense - investment_income_elected_form_4952 - student_loan_interest - educator_expense - qualified_tuition_expenses - casualty_loss - - unreimbursed_business_employee_expenses - - traditional_ira_contributions_desired - id: transfer:person/puf_tax_itemization__batch_3 name: transfer:person/puf_tax_itemization__batch_3 kind: late_transfer @@ -19205,15 +19420,6 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] - - alternatives: - - - column: estate_income_would_be_qualified - entity: person - value_kind: non_null - column: estate_income_would_be_qualified - entity: person - producing_stage: primary_puf_qrf - required_scope: puf_clone - tolerated_absence_receipts: [] - alternatives: - - column: farm_income entity: person @@ -19223,15 +19429,6 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] - - alternatives: - - - column: farm_operations_income_would_be_qualified - entity: person - value_kind: non_null - column: farm_operations_income_would_be_qualified - entity: person - producing_stage: primary_puf_qrf - required_scope: puf_clone - tolerated_absence_receipts: [] - alternatives: - - column: farm_rent_income entity: person @@ -19475,6 +19672,33 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] + - alternatives: + - - column: traditional_ira_contributions_desired + entity: person + value_kind: finite_numeric + column: traditional_ira_contributions_desired + entity: person + producing_stage: source:with_us_retirement_contribution_inputs + required_scope: asec_source + tolerated_absence_receipts: [] + - alternatives: + - - column: traditional_ira_contributions_desired + entity: person + value_kind: finite_numeric + column: traditional_ira_contributions_desired + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] + - alternatives: + - - column: unreimbursed_business_employee_expenses + entity: person + value_kind: finite_numeric + column: unreimbursed_business_employee_expenses + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] - alternatives: - - column: spm_unit_id entity: spm_unit @@ -19607,14 +19831,14 @@ producer_graph: entity: person family: puf_tax_itemization__batch_3 ordered_targets: + - unreimbursed_business_employee_expenses + - traditional_ira_contributions_desired - self_employed_pension_contributions_desired - estate_income - farm_income - farm_rent_income - partnership_income - partnership_self_employment_net_earnings - - estate_income_would_be_qualified - - farm_operations_income_would_be_qualified - id: transfer:person/puf_tax_itemization__batch_4 name: transfer:person/puf_tax_itemization__batch_4 kind: late_transfer @@ -20242,6 +20466,24 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] + - alternatives: + - - column: estate_income_would_be_qualified + entity: person + value_kind: non_null + column: estate_income_would_be_qualified + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] + - alternatives: + - - column: farm_operations_income_would_be_qualified + entity: person + value_kind: non_null + column: farm_operations_income_would_be_qualified + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] - alternatives: - - column: farm_rent_income_would_be_qualified entity: person @@ -20359,15 +20601,6 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: whole_pool tolerated_absence_receipts: [] - - alternatives: - - - column: qualified_bdc_income - entity: person - value_kind: finite_numeric - column: qualified_bdc_income - entity: person - producing_stage: primary_puf_qrf - required_scope: puf_clone - tolerated_absence_receipts: [] - alternatives: - - column: qualified_dividend_income entity: person @@ -20377,15 +20610,6 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] - - alternatives: - - - column: qualified_reit_and_ptp_income - entity: person - value_kind: finite_numeric - column: qualified_reit_and_ptp_income - entity: person - producing_stage: primary_puf_qrf - required_scope: puf_clone - tolerated_absence_receipts: [] - alternatives: - - column: rental_income entity: person @@ -20635,14 +20859,14 @@ producer_graph: entity: person family: puf_tax_itemization__batch_4 ordered_targets: + - estate_income_would_be_qualified + - farm_operations_income_would_be_qualified - farm_rent_income_would_be_qualified - partnership_s_corp_income_would_be_qualified - rental_income_would_be_qualified - self_employment_income_would_be_qualified - sstb_self_employment_income_would_be_qualified - business_is_sstb - - qualified_bdc_income - - qualified_reit_and_ptp_income - id: transfer:person/puf_tax_itemization__batch_5 name: transfer:person/puf_tax_itemization__batch_5 kind: late_transfer @@ -21360,6 +21584,15 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: whole_pool tolerated_absence_receipts: [] + - alternatives: + - - column: qualified_bdc_income + entity: person + value_kind: finite_numeric + column: qualified_bdc_income + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] - alternatives: - - column: qualified_dividend_income entity: person @@ -21369,6 +21602,15 @@ producer_graph: producing_stage: primary_puf_qrf required_scope: puf_clone tolerated_absence_receipts: [] + - alternatives: + - - column: qualified_reit_and_ptp_income + entity: person + value_kind: finite_numeric + column: qualified_reit_and_ptp_income + entity: person + producing_stage: primary_puf_qrf + required_scope: puf_clone + tolerated_absence_receipts: [] - alternatives: - - column: rental_income entity: person @@ -21636,6 +21878,8 @@ producer_graph: entity: person family: puf_tax_itemization__batch_5 ordered_targets: + - qualified_bdc_income + - qualified_reit_and_ptp_income - sstb_self_employment_income_before_lsr - sstb_unadjusted_basis_qualified_property - sstb_w2_wages_from_qualified_business @@ -33464,7 +33708,7 @@ producer_graph: - producer: source:with_us_retirement_contribution_inputs owns_final: true action: final_write - - producer: transfer:person/puf_tax_itemization__batch_2 + - producer: transfer:person/puf_tax_itemization__batch_3 owns_final: false action: producer_masked_byte_exact_noop - entity: person @@ -33480,7 +33724,7 @@ producer_graph: - producer: source:with_us_retirement_contribution_inputs owns_final: true action: final_write - - producer: transfer:person/puf_tax_itemization__batch_2 + - producer: transfer:person/puf_tax_itemization__batch_3 owns_final: false action: producer_masked_byte_exact_noop - entity: person @@ -33496,14 +33740,14 @@ producer_graph: - producer: source:with_us_retirement_contribution_inputs owns_final: true action: byte_exact_clone_1_mirror - - producer: transfer:person/puf_tax_itemization__batch_2 + - producer: transfer:person/puf_tax_itemization__batch_3 owns_final: false action: producer_masked_byte_exact_noop - entity: person target: traditional_ira_contributions_desired origin: acs clone_index: 0 - final_owner: transfer:person/puf_tax_itemization__batch_2 + final_owner: transfer:person/puf_tax_itemization__batch_3 finalization: late_transfer_owner_last producer_actions: - producer: primary_puf_qrf @@ -33512,7 +33756,7 @@ producer_graph: - producer: source:with_us_retirement_contribution_inputs owns_final: false action: origin_projection_masked_noop - - producer: transfer:person/puf_tax_itemization__batch_2 + - producer: transfer:person/puf_tax_itemization__batch_3 owns_final: true action: final_write - entity: person @@ -33528,7 +33772,7 @@ producer_graph: - producer: source:with_us_retirement_contribution_inputs owns_final: false action: origin_projection_masked_noop - - producer: transfer:person/puf_tax_itemization__batch_2 + - producer: transfer:person/puf_tax_itemization__batch_3 owns_final: false action: producer_masked_byte_exact_noop - entity: person @@ -33544,7 +33788,7 @@ producer_graph: - producer: source:with_us_retirement_contribution_inputs owns_final: false action: origin_projection_masked_noop - - producer: transfer:person/puf_tax_itemization__batch_2 + - producer: transfer:person/puf_tax_itemization__batch_3 owns_final: false action: producer_masked_byte_exact_noop - entity: person diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index e33ffec848..5bda65a467 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf + sha256: 9e807e4bb83ded40c177c130515faad2fdcde747c6acb825e0d601e1609538b7 stage_manifest: version: 1 country: us @@ -242,8 +242,13 @@ stages: published_source_year: 2015 published_total_column: CF published_home_mortgage_column: CH + published_deductible_points_column: CN + published_mortgage_insurance_premiums_column: CP + published_investment_interest_column: CR mortgage_output: home_mortgage_interest - non_mortgage_output: investment_interest_expense + points_output: home_mortgage_points + premiums_output: mortgage_insurance_premiums + investment_output: investment_interest_expense allocation: deterministic proportional allocation within each published AGI band outlier_screen_before_split: variable: home_mortgage_interest @@ -284,6 +289,8 @@ stages: - charitable_non_cash_donations - real_estate_taxes - home_mortgage_interest + - home_mortgage_points + - mortgage_insurance_premiums - investment_interest_expense - investment_income_elected_form_4952 - student_loan_interest @@ -348,6 +355,8 @@ stages: - charitable_non_cash_donations - real_estate_taxes - home_mortgage_interest + - home_mortgage_points + - mortgage_insurance_premiums - investment_interest_expense - investment_income_elected_form_4952 - student_loan_interest @@ -2407,7 +2416,7 @@ stages: notes: This structural stage consumes the mortgage-only lineage already produced by puf_tax_detail's source-backed Table 2.1 E19200 decomposition. It combines that lineage with SCF mortgage-debt hints to derive mortgage balances and person-carried mortgage inputs; it neither constructs nor overwrites - investment_interest_expense. + home_mortgage_points, mortgage_insurance_premiums, or investment_interest_expense. - stage: acs_rent survey: Census ACS 2022 source: https://www.census.gov/programs-surveys/acs diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 74e4757cba..c6dd47698d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -572,9 +572,11 @@ from microcosm.build.us_runtime.puf_interest_components import ( US_PUF_E19200_AGI_BANDS, US_PUF_E19200_ALL_RETURNS_COMPONENTS, + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, PufE19200AgiBand, PufE19200InterestComponents, split_us_puf_e19200_by_agi_band, + split_us_puf_e19200_residual_by_agi_band, ) from microcosm.build.us_runtime.puf_source_agi import ( PUF_AGGREGATE_DISAGGREGATION_SEED, @@ -1871,6 +1873,7 @@ "US_PUF_DONOR_MORTGAGE_OUTLIER_CEILING", "US_PUF_E19200_AGI_BANDS", "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS", "PUF_AGGREGATE_DISAGGREGATION_SEED", "PUF_AGGREGATE_RECIDS", "PUF_SOURCE_YEAR", @@ -1922,6 +1925,7 @@ "reform_validation_payload", "source_gap_family_ids", "split_us_puf_e19200_by_agi_band", + "split_us_puf_e19200_residual_by_agi_band", "source_year_puf_adjusted_gross_income", "select_puf_capital_gains_tail_donors", "ECPS_PARITY_KNOWN_GAPS_RESOURCE", @@ -2391,7 +2395,7 @@ def to_manifest(self) -> dict[str, object]: notes=( "Itemized-deduction detail, versioned processed-PUF Section 199A " "simulation leaves (carried without redrawing), partnership SE, " - "source-year-AGI E19200 mortgage/non-mortgage split, direct " + "source-year-AGI E19200 mortgage/points/premiums/investment split, direct " "E00800/E03500 alimony, direct E20500 casualty loss, and the E20400 " "miscellaneous-expense proxy. The pinned processed PUF uprates its " "raw TY2015 rows before seeded disclosure-record replacement and " diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py index 1cf1d0ec54..23aa83b044 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/fiscal_targets.py @@ -188,8 +188,10 @@ def _us_hierarchy_seed( # structural carriers are independently imputed, so equality is a data # property, verified on O-1 post-itemizer-mask (identical weighted carrier # totals; the only gross-vs-structural divergent units are non-itemizers). - # The residual overshoot is the donor-side E19200 total-interest lineage - # (microcosm#515, #487-adjacent). + # Donor-side E19200 decomposition now keeps the CH mortgage lineage + # separate from CN points, CP qualified mortgage-insurance premiums and + # CR investment interest. Those residual leaves do not belong in this + # mortgage-only target. "home_mortgage_interest_amount": "deductible_mortgage_interest", } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py b/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py index ce76b43846..f9d5efed6a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py @@ -93,6 +93,9 @@ from microcosm.build.us_runtime.prior_year_income import ( US_PRIOR_YEAR_INCOME_NONCONSTANT_PERSON_COLUMNS, ) +from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, +) from microcosm.build.us_runtime.qbi_inputs import ( US_QBI_NONCONSTANT_PERSON_COLUMNS, ) @@ -196,7 +199,7 @@ *US_CAPITAL_GAIN_DETAILS_NONCONSTANT_PERSON_COLUMNS, *US_SALT_REFUND_NONCONSTANT_PERSON_COLUMNS, *US_HOUSING_NONCONSTANT_PERSON_COLUMNS, - "investment_interest_expense", + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, ) US_RELEASE_REQUIRED_SPM_UNIT_SOURCE_COLUMNS = ( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_interest_components.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_interest_components.py index 5a22f3f5c7..704abb605a 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_interest_components.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_interest_components.py @@ -12,6 +12,11 @@ import numpy as np _SOURCE_ASSET = "soi_table_2_1_interest_components_ty2015.json" +US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS = ( + "home_mortgage_points", + "mortgage_insurance_premiums", + "investment_interest_expense", +) _AMOUNT_COLUMNS = { "total_interest_paid_amount": "CF", "home_mortgage_interest_amount": "CH", @@ -38,7 +43,7 @@ class PufE19200InterestComponents: @property def non_mortgage_interest_amount(self) -> int: - """Return the conserving residual represented by the engine leaf.""" + """Return the conserving residual split among three engine leaves.""" return self.total_interest_paid_amount - self.home_mortgage_interest_amount @@ -66,6 +71,47 @@ def home_mortgage_share(self) -> float: return self.home_mortgage_interest_amount / self.total_interest_paid_amount + @property + def residual_component_amount(self) -> int: + """Return CN + CP + CR, retaining the published source rounding.""" + + return ( + self.deductible_points_amount + + self.qualified_mortgage_insurance_premiums_amount + + self.investment_interest_amount + ) + + @property + def deductible_points_residual_share(self) -> float: + """Return the published points share of the non-mortgage components.""" + + return self.deductible_points_amount / self.residual_component_amount + + @property + def qualified_mortgage_insurance_premiums_residual_share(self) -> float: + """Return the published premiums share of the residual components.""" + + return ( + self.qualified_mortgage_insurance_premiums_amount + / self.residual_component_amount + ) + + @property + def investment_interest_residual_share(self) -> float: + """Return the published investment share of the residual components.""" + + return self.investment_interest_amount / self.residual_component_amount + + @property + def deductible_points_within_points_and_premiums_share(self) -> float: + """Return CN / (CN + CP), explicitly handling absent components.""" + + denominator = ( + self.deductible_points_amount + + self.qualified_mortgage_insurance_premiums_amount + ) + return self.deductible_points_amount / denominator if denominator else 0.0 + def _components(raw: dict[str, Any]) -> PufE19200InterestComponents: return PufE19200InterestComponents( @@ -142,6 +188,15 @@ def _load_source_asset( ) if any(amount < 0 for amount in amounts) or row.total_interest_paid_amount == 0: raise ValueError(f"{_SOURCE_ASSET} carries an invalid amount.") + if ( + row.deductible_points_amount + + row.qualified_mortgage_insurance_premiums_amount + == 0 + ): + raise ValueError( + f"{_SOURCE_ASSET} row {row.source_row} has zero CN + CP; " + "the points-and-premiums split requires a positive denominator." + ) component_sum = sum(amounts[1:]) if abs(row.total_interest_paid_amount - component_sum) > 1: raise ValueError( @@ -177,6 +232,14 @@ def _agi_band_identity(band: PufE19200AgiBand) -> dict[str, object]: "lower_bound": band.lower_bound, "upper_bound": band.upper_bound, "home_mortgage_share": band.home_mortgage_share, + "deductible_points_residual_share": band.deductible_points_residual_share, + "qualified_mortgage_insurance_premiums_residual_share": ( + band.qualified_mortgage_insurance_premiums_residual_share + ), + "investment_interest_residual_share": band.investment_interest_residual_share, + "deductible_points_within_points_and_premiums_share": ( + band.deductible_points_within_points_and_premiums_share + ), } @@ -216,6 +279,25 @@ def puf_e19200_interest_components_asset_identity( [band.home_mortgage_share for band in US_PUF_E19200_AGI_BANDS], dtype=np.float64, ) +_INVESTMENT_RESIDUAL_SHARES = np.asarray( + [band.investment_interest_residual_share for band in US_PUF_E19200_AGI_BANDS], + dtype=np.float64, +) +_POINTS_WITHIN_POINTS_AND_PREMIUMS_SHARES = np.asarray( + [ + band.deductible_points_within_points_and_premiums_share + for band in US_PUF_E19200_AGI_BANDS + ], + dtype=np.float64, +) +_POINTS_AND_PREMIUMS_AMOUNTS = np.asarray( + [ + band.deductible_points_amount + + band.qualified_mortgage_insurance_premiums_amount + for band in US_PUF_E19200_AGI_BANDS + ], + dtype=np.int64, +) def puf_e19200_agi_bands_runtime_identity( @@ -252,12 +334,11 @@ def split_us_puf_e19200_by_agi_band( ) -> tuple[np.ndarray, np.ndarray]: """Split each E19200 record by its published TY2015 SOI AGI-band share. - PolicyEngine has one non-mortgage interest input, - ``investment_interest_expense``. The residual routed there is broader - than the table's investment-interest column: it also carries deductible - points and qualified mortgage-insurance premiums. Computing it as - ``total - mortgage`` preserves E19200 exactly despite published component - rounding. + Computing the non-mortgage residual as ``total - mortgage`` preserves + E19200 exactly despite published component rounding. The residual carries + deductible points, qualified mortgage-insurance premiums and investment + interest; ``split_us_puf_e19200_residual_by_agi_band`` separates those + three leaves before they are routed to their respective engine inputs. """ total = np.asarray(total_interest_paid, dtype=np.float64) @@ -296,11 +377,62 @@ def split_us_puf_e19200_by_agi_band( return mortgage, non_mortgage +def split_us_puf_e19200_residual_by_agi_band( + non_mortgage_interest_paid: Any, + adjusted_gross_income: Any, +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Split the conserved E19200 residual into points, premiums, investment. + + Shares use the same TY2015 AGI bands as the mortgage split. First split + the residual by CR / (CN + CP + CR), then split points-and-premiums by + CN / (CN + CP). Each binary split recomputes its first component as the + complement of the rounded second component, using the Sterbenz-exact + construction proved in ``split_us_puf_e19200_by_agi_band``. Thus + ``(points + premiums) + investment == non_mortgage`` both as an exact + real sum and under that floating-point grouping. Together with the + unchanged mortgage split, ``math.fsum`` recovers E19200 bit-for-bit, + apart from the existing canonicalization of accepted -0.0 to +0.0. + """ + + residual = np.asarray(non_mortgage_interest_paid, dtype=np.float64) + agi = np.asarray(adjusted_gross_income, dtype=np.float64) + if residual.ndim != 1 or agi.ndim != 1: + raise ValueError( + "E19200 residual and adjusted_gross_income must be one-dimensional." + ) + if len(residual) != len(agi): + raise ValueError( + "E19200 residual and adjusted_gross_income must have the same record count." + ) + if not np.isfinite(residual).all(): + raise ValueError("E19200 residual must contain only finite values.") + if not np.isfinite(agi).all(): + raise ValueError("adjusted_gross_income must contain only finite values.") + if (residual < 0).any(): + raise ValueError("E19200 residual must be nonnegative.") + + band_index = np.searchsorted(_AGI_UPPER_BOUNDS, agi, side="right") + investment = residual * _INVESTMENT_RESIDUAL_SHARES[band_index] + points_and_premiums = residual - investment + investment = residual - points_and_premiums + points = points_and_premiums * _POINTS_WITHIN_POINTS_AND_PREMIUMS_SHARES[band_index] + premiums = points_and_premiums - points + points = points_and_premiums - premiums + # The loader rejects such a source row, but make the denominator-zero + # behavior explicit here too: CN = CP = 0 implies no points or premiums. + absent_points_and_premiums = _POINTS_AND_PREMIUMS_AMOUNTS[band_index] == 0 + points[absent_points_and_premiums] = 0.0 + premiums[absent_points_and_premiums] = 0.0 + return points, premiums, investment + + __all__ = [ "PufE19200AgiBand", "PufE19200InterestComponents", "US_PUF_E19200_AGI_BANDS", "US_PUF_E19200_ALL_RETURNS_COMPONENTS", + "US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS", "puf_e19200_interest_components_asset_identity", "split_us_puf_e19200_by_agi_band", + "split_us_puf_e19200_residual_by_agi_band", ] diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py index 616c9352a7..e615d072d6 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_qrf_chain.py @@ -77,7 +77,10 @@ # v6 makes that recipient-universe authority a versioned chain semantic. Every # v1--v5 root or target must rebuild rather than sharing a schema label with a # chain whose root, banks, targets, and finalization bind the added receipt. -PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION = 6 +# v7 splits the E19200 residual into points, mortgage-insurance premiums and +# investment interest. Even a custom-order v6 bank may carry the old whole +# residual as investment interest, so it must rebuild under these semantics. +PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION = 7 PRIMARY_QRF_MANIFEST_FILENAME = "manifest.json" PRIMARY_QRF_DONOR_FILENAME = "donor.frame.h5" PRIMARY_QRF_RECIPIENT_FILENAME = "recipient.frame.h5" @@ -87,7 +90,7 @@ *PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) PRIMARY_QRF_TARGET_ORDER_SHA256 = ( - "795519d161e6b8425fc3b64de7eb435d52d25e7c8250b5861f3bb21ab48266a3" + "53b2ee6e64cb746842d771598a13aba063ad8df2d3437a4cdf69ae440655b3b3" ) _ARTIFACT_KIND = "populace_primary_puf_qrf_chain" diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py index 6d609532a9..bcc18dd732 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/puf_support.py @@ -33,7 +33,9 @@ puf_processed_capital_gains_stage, ) from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, split_us_puf_e19200_by_agi_band, + split_us_puf_e19200_residual_by_agi_band, ) from microcosm.build.us_runtime.qbi_inputs import ( US_QBI_BOOLEAN_OUTPUT_COLUMNS, @@ -178,7 +180,7 @@ ) US_PUF_DONOR_MORTGAGE_QUARANTINE_FIELDS = ( "home_mortgage_interest", - "investment_interest_expense", + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, "first_home_mortgage_interest", "second_home_mortgage_interest", "interest_deduction", @@ -302,7 +304,7 @@ class _PredictorSourcePlan: "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", - "investment_interest_expense", + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -464,7 +466,7 @@ def puf_tax_detail_tail_bound_quantiles_identity() -> dict[str, float]: "charitable_non_cash_donations", "real_estate_taxes", "home_mortgage_interest", - "investment_interest_expense", + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, "investment_income_elected_form_4952", "student_loan_interest", "educator_expense", @@ -1460,6 +1462,14 @@ def puf_tax_unit_donor_from_arrays( if values is not None: tax_unit[output] = values + if _MORTGAGE_OUTLIER_SCREEN_COLUMN in tax_unit: + # The processed PUF predates these residual leaves. Their source is + # raw E19200, so requested leaves may be derived without a stored + # input array; existing values still reach the overwrite guard below. + for output in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + if output in person_outputs and output not in tax_unit: + tax_unit[output] = np.zeros(len(tax_unit), dtype=np.float64) + required_outputs = [*person_outputs, *tax_unit_outputs] missing = [column for column in required_outputs if column not in tax_unit.columns] if missing: @@ -1474,7 +1484,7 @@ def puf_tax_unit_donor_from_arrays( # Threshold the grouped RAW person value: thresholding the carved # mortgage value at the same literal would miss corrupt rows in the # $10M-to-$10.75M raw band. Keep the mask while the E19200 split - # materializes both conserving components, then quarantine only those + # materializes all four conserving leaves, then quarantine only those # implicated fields so unrelated donor values remain available. mortgage_quarantine_mask = ( tax_unit[_MORTGAGE_OUTLIER_SCREEN_COLUMN].to_numpy( @@ -1581,7 +1591,7 @@ def _quarantine_us_puf_mortgage_fields( def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: - """Split raw E19200 into mortgage and modeled non-mortgage components.""" + """Split raw E19200 into mortgage, points, premiums and investment leaves.""" if _MORTGAGE_OUTLIER_SCREEN_COLUMN not in donor: donor.drop(columns=[_E19200_AGI_BAND_COLUMN], errors="ignore", inplace=True) @@ -1596,20 +1606,23 @@ def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: "adjusted_gross_income is required to split nonzero PUF E19200 " "records by the published SOI AGI bands." ) - if "investment_interest_expense" in donor: - existing = donor["investment_interest_expense"].to_numpy( + for column in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + if column not in donor: + continue + existing = donor[column].to_numpy( dtype=np.float64, copy=False, ) if (existing != 0).any(): raise ValueError( - "Processed PUF already carries nonzero investment_interest_expense; " + f"Processed PUF already carries nonzero {column}; " "refusing to overwrite independently sourced values with the " "E19200 residual." ) if not has_nonzero_e19200: - if "investment_interest_expense" in donor: - donor["investment_interest_expense"] = np.zeros_like(raw_total) + for column in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + if column in donor: + donor[column] = np.zeros_like(raw_total) donor.drop( columns=[ _MORTGAGE_OUTLIER_SCREEN_COLUMN, @@ -1620,10 +1633,9 @@ def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: ) return - mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( - raw_total, - donor[_E19200_AGI_BAND_COLUMN].to_numpy(dtype=np.float64, copy=False), - ) + agi = donor[_E19200_AGI_BAND_COLUMN].to_numpy(dtype=np.float64, copy=False) + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band(raw_total, agi) + residual_leaves = split_us_puf_e19200_residual_by_agi_band(non_mortgage, agi) band_share = np.divide( mortgage, raw_total, @@ -1632,11 +1644,20 @@ def _split_us_puf_e19200_components(donor: pd.DataFrame) -> None: ) for column in _US_PUF_E19200_LINEAGE_DONOR_COLUMNS: if column in donor: - donor[column] = ( - donor[column].to_numpy(dtype=np.float64, copy=False) * band_share - ) - if "investment_interest_expense" in donor: - donor["investment_interest_expense"] = non_mortgage + if column == "home_mortgage_interest": + # Keep the original splitter's mortgage bits; re-multiplying + # raw_total by its rounded share can move this leaf by a ULP + # and break the exact four-leaf conservation construction. + donor[column] = mortgage + else: + donor[column] = ( + donor[column].to_numpy(dtype=np.float64, copy=False) * band_share + ) + for column, values in zip( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, residual_leaves, strict=True + ): + if column in donor: + donor[column] = values donor.drop( columns=[ _MORTGAGE_OUTLIER_SCREEN_COLUMN, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index e803bb0137..072c26dbd7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -85,6 +85,9 @@ from microcosm.build.us_runtime.prior_year_income import ( US_PRIOR_YEAR_INCOME_PERSISTED_OUTPUT_COLUMNS, ) +from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, +) from microcosm.build.us_runtime.qbi_inputs import US_QBI_OUTPUT_COLUMNS from microcosm.build.us_runtime.relationship_inputs import ( US_RELATIONSHIP_INPUTS_OUTPUT_COLUMNS, @@ -156,6 +159,10 @@ # become structural zeroes. POST_REFERENCE_ECPS_REQUIRED_INPUTS = frozenset( { + # The frozen eCPS predates the separate Schedule A CN/CP inputs. + # No reference nonzero share exists; the source-backed leaves are + # nevertheless required on future exports, alongside CR. + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS[:2], "fsla_overtime_premium", "qualified_passenger_vehicle_loan_interest", "traditional_401k_contributions_desired", @@ -881,6 +888,12 @@ def assert_release_input_coverage_manifest_current( failures.append( f"{column}: restored reference eCPS input must remain required." ) + for column in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + if column not in required or column in reviewed: + failures.append( + f"{column}: source-backed E19200 leaf must remain required " + "without a reviewed exclusion." + ) for column in sorted( RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS & parity_gap_layers.keys() ): diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py index d71320311f..2d7596fd0b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/stacked_spine.py @@ -2630,6 +2630,8 @@ def _terminal_surface_from_pool_registry() -> TargetFamilies: "charitable_non_cash_donations", ), ("person", "puf_tax_itemization", "home_mortgage_interest"), + ("person", "puf_tax_itemization", "home_mortgage_points"), + ("person", "puf_tax_itemization", "mortgage_insurance_premiums"), ( "person", "puf_tax_itemization", @@ -5062,7 +5064,7 @@ def validate_stacked_post_puf_transfer_receipt( if not isinstance(targets, Mapping) or set(targets) != expected_target_labels: raise ValueError( f"{boundary}: stacked post-PUF transfer target surface is not the " - "canonical 70-target surface; production manifest emission is " + "canonical 72-target surface; production manifest emission is " "forbidden." ) for name, group in expected_groups.items(): @@ -5095,7 +5097,7 @@ def validate_stacked_post_puf_transfer_receipt( if completion != { "status": "complete", "group_count": 19, - "target_count": 70, + "target_count": 72, "residual_null_rows": 0, }: raise ValueError( @@ -8979,11 +8981,11 @@ def validate_structural_absence_receipt( _surface_target_keys(_freeze_target_families(pool_transfer_target_families())) ) if ( - len(_canonical_surface_keys) != 134 - or len(set(_canonical_surface_keys)) != 134 + len(_canonical_surface_keys) != 136 + or len(set(_canonical_surface_keys)) != 136 or len(_canonical_early_transfer_keys) != 48 - or len(_canonical_late_transfer_keys) != 70 - or len(_canonical_late_puf_producer_keys) != 43 + or len(_canonical_late_transfer_keys) != 72 + or len(_canonical_late_puf_producer_keys) != 45 or len(_canonical_late_source_producer_keys) != 29 or len(_canonical_late_puf_producer_keys & _canonical_late_source_producer_keys) != 2 @@ -8992,7 +8994,7 @@ def validate_structural_absence_receipt( or _canonical_early_transfer_keys & _canonical_late_transfer_keys or _canonical_early_transfer_keys | _canonical_late_transfer_keys != _canonical_full_transfer_keys - or len(_canonical_full_transfer_keys) != 118 + or len(_canonical_full_transfer_keys) != 120 or set(_plan_target_keys(_CANONICAL_STACKED_GAP_FILL_PLAN_ANCHOR)) != _canonical_early_transfer_keys or not _canonical_full_transfer_keys.issubset(_canonical_surface_keys) @@ -9012,10 +9014,10 @@ def validate_structural_absence_receipt( ) ): raise RuntimeError( - "Canonical stacked authority must partition the exact 118-target " - "transfer surface into 48 early gap-fill and 70 post-PUF targets " - "inside an exact 134-target terminal surface and metric registry; " - "the late surface must be exactly covered by 43 PUF-clone and 29 " + "Canonical stacked authority must partition the exact 120-target " + "transfer surface into 48 early gap-fill and 72 post-PUF targets " + "inside an exact 136-target terminal surface and metric registry; " + "the late surface must be exactly covered by 45 PUF-clone and 29 " "ASEC-source producer targets with their declared two-target overlap." ) @@ -11137,7 +11139,7 @@ def _aggregate_late_transfer_result( ], execution_order: Sequence[str], ) -> StackedPostPufTransferResult: - """Bind all bounded group outcomes into the canonical 70-target receipt.""" + """Bind all bounded group outcomes into the canonical 72-target receipt.""" expected_groups = tuple(group.name for group in CANONICAL_US_LATE_TRANSFER_GROUPS) if set(group_results) != set(expected_groups): @@ -13406,7 +13408,7 @@ class OriginBatterySpec: """Test-seam grouping for per-column battery metrics. Production never accepts these specs from a caller: it consumes the - immutable 134-column canonical registry. The explicit test-authority seam + immutable 136-column canonical registry. The explicit test-authority seam groups its digested registry into specs so the comparison engine can reuse the same loop. ``clone_index`` scopes a fixture comparison to one clone role: 0 compares native rows and 1 compares a PUF arm. @@ -13460,7 +13462,7 @@ def by_origin_battery( *, tail_manifest: Mapping[str, object] | None = None, ) -> GateResult: - """Run the canonical 134-target plus joint by-origin battery.""" + """Run the canonical 136-target plus joint by-origin battery.""" return _by_origin_battery_evaluate( frame, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_overlap_ownership.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_overlap_ownership.py index bbcb580f1f..413f52e75f 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_overlap_ownership.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_overlap_ownership.py @@ -27,7 +27,7 @@ _EDUCATION_SOURCE_PRODUCER = "source:with_us_education_inputs" _RETIREMENT_SOURCE_PRODUCER = "source:with_us_retirement_contribution_inputs" _TUITION_TRANSFER_PRODUCER = "transfer:person/puf_tax_itemization__batch_2" -_TRADITIONAL_IRA_TRANSFER_PRODUCER = "transfer:person/puf_tax_itemization__batch_2" +_TRADITIONAL_IRA_TRANSFER_PRODUCER = "transfer:person/puf_tax_itemization__batch_3" _SELF_EMPLOYED_PENSION_TRANSFER_PRODUCER = ( "transfer:person/puf_tax_itemization__batch_3" ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py index 134ec403e7..d6b2f1a308 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/us_late_producer_registry.py @@ -1412,11 +1412,11 @@ def _bounded_transfer_groups( ) if ( len(CANONICAL_US_LATE_TRANSFER_GROUPS) != 19 - or sum(len(group.targets) for group in CANONICAL_US_LATE_TRANSFER_GROUPS) != 70 + or sum(len(group.targets) for group in CANONICAL_US_LATE_TRANSFER_GROUPS) != 72 ): raise RuntimeError( "Canonical US late transfer must contain exactly 19 bounded groups " - "and 70 ordered targets." + "and 72 ordered targets." ) _canonical_late_targets = { target for group in CANONICAL_US_LATE_TRANSFER_GROUPS for target in group.targets @@ -1425,11 +1425,11 @@ def _bounded_transfer_groups( len(_BOOLEAN_LATE_TARGETS) != 17 or len(_STRING_LATE_TARGETS) != 2 or not (_BOOLEAN_LATE_TARGETS | _STRING_LATE_TARGETS) <= _canonical_late_targets - or len(_canonical_late_targets - _BOOLEAN_LATE_TARGETS - _STRING_LATE_TARGETS) != 51 + or len(_canonical_late_targets - _BOOLEAN_LATE_TARGETS - _STRING_LATE_TARGETS) != 53 ): raise RuntimeError( - "Canonical US late target kinds must partition 70 targets into " - "51 numeric, 17 boolean, and 2 string inputs." + "Canonical US late target kinds must partition 72 targets into " + "53 numeric, 17 boolean, and 2 string inputs." ) US_LATE_TRANSFER_INPUT_INVENTORIES: Mapping[str, SourceInputInventory] = ( MappingProxyType( diff --git a/packages/microcosm-build/tests/engine_contract/us/test_imputation_lineage_spec.py b/packages/microcosm-build/tests/engine_contract/us/test_imputation_lineage_spec.py index 84ff79dc76..ea8b9e82c3 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_imputation_lineage_spec.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_imputation_lineage_spec.py @@ -133,8 +133,8 @@ def test_complete_bundle_family_inventory(spec: dict[str, object]) -> None: } assert targets == { "gap_fill_stacked_spine": 48, - "primary_puf_qrf": 65, - "late_producer_dag": 70, + "primary_puf_qrf": 67, + "late_producer_dag": 72, } for family in spec["families"]: keys = [(target["entity"], target["name"]) for target in family["targets"]] @@ -145,12 +145,12 @@ def test_dashboard_compiles_the_packaged_bundle_without_a_mirror_spec() -> None: payload = emit() assert payload["spec_binding"]["spec_sha256"] == payload["spec_sha256"] assert payload["counts"] == { - "imputed_variables": 183, + "imputed_variables": 187, "families": 33, "computed_producers": 38, "boolean": 45, - "amount": 132, + "amount": 136, "categorical": 5, - "value_kinds": {"amount": 132, "category": 5, "count": 1, "flag": 45}, + "value_kinds": {"amount": 136, "category": 5, "count": 1, "flag": 45}, } assert len(payload["known_gaps"]) == 1 diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 7be8062b02..ad24b2248e 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_184 - assert fields["authored_normative_field_count"] == 32_404 + assert fields["configuration_field_count"] == 42_457 + assert fields["authored_normative_field_count"] == 32_677 assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_184 + assert fields["consumed_field_count"] == 42_457 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_055, + "compiler_semantic": 27_945, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_751, + "no_generation0_effect": 3_706, } inventory = coverage_report["inventory_coverage"] @@ -105,9 +105,9 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["covered_item_count"] == 41 assert inventory["missing_item_count"] == 0 assert inventory["missing_items"] == [] - assert inventory["counts"]["producer_inputs"] == 2_744 + assert inventory["counts"]["producer_inputs"] == 2_748 assert inventory["counts"]["ownership_rows"] == 18 - assert inventory["counts"]["tail_control_fields"] == 934 + assert inventory["counts"]["tail_control_fields"] == 1112 assert inventory["counts"]["seed_owner_bindings"] == 112 diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py index 95258db5a7..284f3f613f 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py @@ -70,7 +70,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer assert projected == live assert stacked_identity_bytes(projected) == _canonical_bytes(live) assert projected["sha256"] == ( - "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "7e55d1f4b099efdb36e2127354d6ffe77678b2c42c772b150bc19265b6937aad" ) assert { name: component["sha256"] for name, component in projected["components"].items() @@ -188,7 +188,7 @@ def test_checkpoint_projection_is_field_and_byte_identical_to_live_oracle( ] assert ( projected["pool_code"]["late_producer_schedule"]["schedule_sha256"] - == "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + == "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" ) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_typed_closure.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_typed_closure.py index 05f349a724..dc5125155b 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_typed_closure.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_typed_closure.py @@ -51,7 +51,7 @@ def test_us_typed_closure_has_exact_inventory_counts( typed_resources: dict[str, Any], typed_result: TypedClosureResult, ) -> None: - assert len(typed_result.columns) == 176 + assert len(typed_result.columns) == 178 assert len(typed_result.artifacts) == 84 assert len(typed_result.scopes) == 7 assert Counter(artifact.kind for artifact in typed_result.artifacts) == { @@ -63,7 +63,7 @@ def test_us_typed_closure_has_exact_inventory_counts( graph = typed_resources["imputation"]["producer_graph"] assert sum(len(node["outputs"]) for node in graph["nodes"]) == 92 compiled_outputs = compile_producer_outputs(typed_resources) - assert sum(len(rows) for rows in compiled_outputs.values()) == 227 + assert sum(len(rows) for rows in compiled_outputs.values()) == 231 assert sum(len(node["virtual_resources"]) for node in graph["nodes"]) == 75 column_keys = {column.key for column in typed_result.columns} output_artifact_ids = { diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_contract/us/test_us_bundle_core_contracts.py index 64ede4c777..ad3e829beb 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_bundle_core_contracts.py @@ -8,8 +8,8 @@ def test_us_catalog_has_complete_explicit_contracts() -> None: __import__("policyengine_us") catalog = build_catalogs() columns = catalog["columns"] - assert len(columns) == 176 - assert len({row["key"] for row in columns}) == 176 + assert len(columns) == 178 + assert len({row["key"] for row in columns}) == 178 assert catalog["metadata_waivers"] == [ { "id": "policyengine_us_unit_unavailable", @@ -34,7 +34,7 @@ def test_us_catalog_has_complete_explicit_contracts() -> None: else: assert "unit_waiver" not in contract assert Counter(row["contract"]["unit"] for row in columns) == { - "unit_not_declared_by_engine_metadata": 89, + "unit_not_declared_by_engine_metadata": 91, "boolean": 51, "count": 27, "categorical": 9, @@ -60,7 +60,7 @@ def test_us_catalog_has_complete_explicit_contracts() -> None: "catalogs": catalog, } ) - assert len(resolved.columns) == 176 + assert len(resolved.columns) == 178 by_key = {row["key"]: row["contract"] for row in columns} for column in resolved.columns: contract = by_key[column.key] diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py b/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py index 01b2e92023..7e1e3fd5a4 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_spec_bundle.py @@ -192,7 +192,7 @@ def test_typed_imputation_reconstructs_all_constants_authority_components( assert len(payloads["overlap_ownership"]["ownership"]) == 18 assert ( payloads["overlap_ownership"]["sha256"] - == "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" + == "d2a54260c1a4da54a247dcd723134c3b6de5cdaef021fc241d77b9e89d7dd675" ) @@ -310,6 +310,13 @@ def collect_sha256(value: object, path: tuple[str, ...] = ()) -> None: "runtime_agi_bands" ] assert set(runtime_bands) == {"schema_version", "agi_bands"} + for band in runtime_bands["agi_bands"]: + assert { + "deductible_points_residual_share", + "qualified_mortgage_insurance_premiums_residual_share", + "investment_interest_residual_share", + "deductible_points_within_points_and_premiums_share", + } <= band.keys() @pytest.mark.parametrize("asset_pin", ["share_asset", "soi_agi_bands"]) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py b/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py index 7e5838e1ab..0f8b926551 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_release_input_coverage.py @@ -271,6 +271,32 @@ def test_restored_investment_interest_is_required_without_exclusion(self) -> Non assert column in manifest.required_columns assert column not in manifest.reviewed_exclusions + def test_e19200_new_leaves_are_required_without_reference_shares(self) -> None: + from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, + ) + from microcosm.build.us_runtime.release_input_coverage import ( + POST_REFERENCE_ECPS_REQUIRED_INPUTS, + ) + + manifest = load_release_input_coverage_manifest() + reference = json.loads( + files("microcosm.build.us") + .joinpath("ecps_parity_reference.json") + .read_text(encoding="utf-8") + ) + for column in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS[:2]: + assert column in POST_REFERENCE_ECPS_REQUIRED_INPUTS + assert column in manifest.required_columns + assert column not in manifest.reviewed_exclusions + assert column not in reference["nonzero_shares"] + assert ( + column + in reference["post_reference_inputs_without_nonzero_share_reference"] + ) + entry = next(entry for entry in manifest.columns if entry.name == column) + assert "No frozen eCPS" in entry.note + def test_post_reference_ssi_disability_criterion_has_unique_probe(self) -> None: manifest = load_release_input_coverage_manifest() column = "meets_ssi_disability_criteria" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_battery_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_battery_semantics.py index ca1b720aaf..a6d0589784 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_battery_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_battery_semantics.py @@ -52,12 +52,12 @@ def test_battery_registry_views_match_live_registry( ) -> None: views = derive_battery_registry_views(battery) assert views["declared_surface"] == _json_ready(CANONICAL_STACKED_DECLARED_SURFACE) - assert views["required_scalar_targets"] == 134 + assert views["required_scalar_targets"] == 136 assert views["required_joint_targets"] == 1 assert views["metric_counts"] == { "boolean_incidence": 51, "categorical_tvd": 4, - "monetary_sign_separated": 79, + "monetary_sign_separated": 81, } @@ -129,7 +129,7 @@ def test_battery_legacy_projection_survives_resolved_normalization( assert projected["gates"][1]["metric"]["params"] == { **battery["gates"][1]["metric"]["params"], "required_joint_targets": 1, - "required_scalar_targets": 134, + "required_scalar_targets": 136, } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py index ccec9ee148..d24349a131 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_calibration_semantics.py @@ -31,7 +31,7 @@ ROOT = _TEST_PATHS.repository US_ROOT = ROOT / "packages/microcosm-build/src/microcosm/build/us" CANONICAL_LEGACY_PROJECTION_SHA256 = ( - "05cb099ee8bd0234ae94586b76dc20ab8a5bd0735c00d4e82e67f912e0b33823" + "b80905b62a2fe9a257f4d985972b01ff6671365da188990363357ac24811831a" ) @@ -215,3 +215,48 @@ def test_tail_reference_mutation_is_refused() -> None: spine_document=resolved.domain("spine").to_wire(), imputation_document=resolved.domain("imputation").to_wire(), ) + + +@pytest.mark.parametrize( + "share", + [ + "deductible_points_residual_share", + "qualified_mortgage_insurance_premiums_residual_share", + "investment_interest_residual_share", + "deductible_points_within_points_and_premiums_share", + ], +) +def test_residual_share_mutation_changes_calibration_runtime_identity( + share: str, +) -> None: + resolved = load_country_spec("us").resolved_spec + assert resolved is not None + calibration_document = resolved.domain("calibration").to_wire() + spine_document = resolved.domain("spine").to_wire() + imputation_document = resolved.domain("imputation").to_wire() + primary = next( + node + for node in imputation_document["producer_graph"]["nodes"] + if node["name"] == "primary_puf_qrf" + ) + resource = next( + row + for row in primary["virtual_resources"] + if row["id"] == "tax_unit.@primary_puf_execution_config" + ) + soi = resource["binding"]["capital_gains_tail"]["soi_e19200_agi_bands"] + before = resolve_calibration_tail_contracts( + calibration_document, + spine_document=spine_document, + imputation_document=imputation_document, + )["puf_capital_gains_tail"]["soi_e19200_agi_bands"] + for rows in (soi["agi_bands"], soi["runtime_agi_bands"]["agi_bands"]): + rows[0][share] /= 2 + after = resolve_calibration_tail_contracts( + calibration_document, + spine_document=spine_document, + imputation_document=imputation_document, + )["puf_capital_gains_tail"]["soi_e19200_agi_bands"] + + assert before["runtime_sha256"] != after["runtime_sha256"] + assert before["asset_sha256"] == after["asset_sha256"] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py index b025205ae5..c294767765 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py @@ -22,7 +22,7 @@ ) from microcosm.build.spec_engine.resolver import F0_CONTRACT_ONLY_KERNEL_IDS -US_SCHEDULE_SHA256 = "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" +US_SCHEDULE_SHA256 = "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" @pytest.fixture(scope="module") @@ -140,10 +140,10 @@ def test_us_node_slices_are_transitive_and_content_attested( def test_ir_retains_all_typed_resolution_products(compiled_us: CompiledSpecIR) -> None: inventory = compiled_us.typed_inventory assert len(inventory["entities"]) == 8 - assert len(inventory["columns"]) == 176 + assert len(inventory["columns"]) == 178 assert len(inventory["artifacts"]) == 84 assert len(inventory["scopes"]) == 7 - assert len(inventory["references"]) == 334 + assert len(inventory["references"]) == 336 assert "engine_abi_lock" in compiled_us.generated_authorities assert "records" in compiled_us.vintage_authorities assert set(compiled_us.surfaces) == { diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index c5ec18bb51..9b150b5514 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_184 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_457 assert field_ledger.source_counts == { - "authored": 32_404, + "authored": 32_677, "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_055, + "compiler_semantic": 27_945, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_751, + "no_generation0_effect": 3_706, } - assert len({field.pointer for field in field_ledger.fields}) == 42_184 + assert len({field.pointer for field in field_ledger.fields}) == 42_457 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py index c9ae76814f..3641a9ed13 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_imputation_semantics.py @@ -98,10 +98,10 @@ def test_imputation_projector_matches_live_plans_and_graph_receipts( for key, expected in live.items(): assert canonical_json_bytes(projected[key]) == canonical_json_bytes(expected) assert projected["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" ) assert projected["overlap_ownership"]["sha256"] == ( - "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" + "d2a54260c1a4da54a247dcd723134c3b6de5cdaef021fc241d77b9e89d7dd675" ) @@ -119,7 +119,7 @@ def test_imputation_projector_matches_live_primary_chain( "checkpoint_schema_version": PRIMARY_QRF_CHECKPOINT_SCHEMA_VERSION, } tuples = derive_primary_effective_predictor_tuples(us_domains["imputation"]) - assert len(tuples) == 65 + assert len(tuples) == 67 preceding: list[str] = [] for row, target in zip(tuples, PRIMARY_QRF_TARGET_ORDER, strict=True): assert row["target"] == target diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py index ab44443757..f7fb9a4ca1 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_legacy_adapter.py @@ -197,13 +197,13 @@ def test_adapter_preserves_generation_zero_identity_components( imputation = legacy_payload["imputation"] assert isinstance(imputation, dict) assert legacy_payload["stacked_authority_receipt"]["sha256"] == ( - "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2" + "7e55d1f4b099efdb36e2127354d6ffe77678b2c42c772b150bc19265b6937aad" ) assert imputation["late_producer_schedule_receipt"]["schedule_sha256"] == ( - "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" ) assert imputation["overlap_ownership"]["sha256"] == ( - "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356" + "d2a54260c1a4da54a247dcd723134c3b6de5cdaef021fc241d77b9e89d7dd675" ) assert legacy_payload["take_up_contract_identity"]["resource_sha256"] == ( "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 718f4ff565..a18768cea8 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "6273f82f0e4899e97d1bede4db0662829b3df6f420dba32c0e67018d010c2c91" + "42b159298a231023fcfd014b017ff6d902b3f5a69f4ed06d34abb385ceec877a" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py b/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py index 924212e93a..971560f9df 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py +++ b/packages/microcosm-build/tests/engine_free/us/test_puf_qrf_chain.py @@ -167,7 +167,9 @@ def test_primary_qrf_production_target_order_is_locked() -> None: *PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS, ) assert PRIMARY_QRF_TARGET_ORDER == expected - assert len(PRIMARY_QRF_TARGET_ORDER) == 65 + assert len(PRIMARY_QRF_TARGET_ORDER) == 67 + assert "home_mortgage_points" in PRIMARY_QRF_TARGET_ORDER + assert "mortgage_insurance_premiums" in PRIMARY_QRF_TARGET_ORDER assert "investment_interest_expense" in PRIMARY_QRF_TARGET_ORDER digest = hashlib.sha256( json.dumps(list(PRIMARY_QRF_TARGET_ORDER), separators=(",", ":")).encode() @@ -688,7 +690,7 @@ def test_primary_qrf_chain_legacy_doctrine_defaults_remain_loadable( n_estimators=2, seed=3, ) - assert manifest["schema_version"] == 6 + assert manifest["schema_version"] == 7 assert "require_complete_recipient_predictors" not in manifest assert "absent_cells" not in manifest assert "recipient_predictor_universe" not in manifest @@ -771,14 +773,14 @@ def test_primary_qrf_finalization_rejects_changed_recipient_features( finalize_primary_puf_qrf_chain(frame, checkpoint_dir) -@pytest.mark.parametrize("stale_version", (1, 2, 3, 4, 5)) +@pytest.mark.parametrize("stale_version", (1, 2, 3, 4, 5, 6)) def test_primary_qrf_rejects_every_stale_schema_version( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, stale_version: int, ) -> None: - # Roots and target checkpoints must reject every schema predating the v6 - # recipient-universe authority. In particular, the v5 case models the old + # Roots and target checkpoints must reject every schema predating the v7 + # three-leaf E19200 split. In particular, the v5 case models the old # strict two-control payload that omitted recipient_predictor_universe; its # stale schema is rejected before that payload can be interpreted. monkeypatch.setenv("POPULACE_FIT_N_JOBS", "1") @@ -813,12 +815,12 @@ def test_primary_qrf_rejects_every_stale_schema_version( manifest_path.write_text(json.dumps(stale_manifest)) with pytest.raises( ValueError, - match=rf"schema version: expected 6, got {stale_version}", + match=rf"schema version: expected 7, got {stale_version}", ): load_primary_puf_qrf_predictions(checkpoint_dir) with pytest.raises( ValueError, - match=rf"schema version: expected 6, got {stale_version}", + match=rf"schema version: expected 7, got {stale_version}", ): run_primary_puf_qrf_chain(checkpoint_dir) manifest_path.write_text(json.dumps(original_manifest)) @@ -841,7 +843,7 @@ def test_primary_qrf_rejects_every_stale_schema_version( ) with pytest.raises( ValueError, - match=rf"invalid schema_version: expected 6, got {stale_version}", + match=rf"invalid schema_version: expected 7, got {stale_version}", ): load_primary_puf_qrf_predictions(checkpoint_dir) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_buildj_base_script.py b/packages/microcosm-build/tests/engine_free/us/test_us_buildj_base_script.py index 6721d4f80c..68ac42afac 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_buildj_base_script.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_buildj_base_script.py @@ -39,6 +39,8 @@ def test_cache_rejects_bases_predating_measured_medicare_take_up() -> None: required = _cache_required_columns() assert "investment_interest_expense" in required["person"] + assert "home_mortgage_points" in required["person"] + assert "mortgage_insurance_premiums" in required["person"] assert "takes_up_medicare_if_eligible" in required["person"] assert "workers_compensation" in required["person"] assert "takes_up_wic_if_eligible" in required["person"] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index a780c4814b..d692d52c7f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf", + "sha256": "9e807e4bb83ded40c177c130515faad2fdcde747c6acb825e0d601e1609538b7", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py index ce708331ae..00418cca96 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py @@ -70,6 +70,8 @@ def _us_frame(**person_extra: object) -> Frame: "casualty_loss": [0.0, 2_500.0, 0.0], "unreimbursed_business_employee_expenses": [1_200.0, 0.0, 800.0], "investment_income_elected_form_4952": [0.0, 500.0, 250.0], + "home_mortgage_points": [0.0, 700.0, 300.0], + "mortgage_insurance_premiums": [0.0, 100.0, 400.0], "investment_interest_expense": [0.0, 800.0, 200.0], "salt_refund_income": [0.0, 1_200.0, 400.0], "long_term_capital_gains_on_collectibles": [0.0, 2_500.0, 1_000.0], @@ -645,12 +647,15 @@ def test_required_us_release_source_columns_enforces_form_4952_signal() -> None: assert_required_us_release_source_columns(raw_frame) -def test_required_us_release_source_columns_enforces_investment_interest_signal() -> ( - None -): +@pytest.mark.parametrize( + "column", l0_refit_export.US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS +) +def test_required_us_release_source_columns_enforces_e19200_residual_leaf_signal( + column: str, +) -> None: frame = _us_frame() raw_people = frame.table("person").copy() - raw_people["investment_interest_expense"] = 0.0 + raw_people[column] = 0.0 raw_frame = Frame( { **{entity: frame.table(entity).copy() for entity in frame.schema.entities}, @@ -662,7 +667,7 @@ def test_required_us_release_source_columns_enforces_investment_interest_signal( with pytest.raises( ValueError, - match="person.investment_interest_expense: not nonconstant", + match=rf"person\.{column}: not nonconstant", ): assert_required_us_release_source_columns(raw_frame) @@ -1090,7 +1095,8 @@ def write_dataset(self, bundle, path, period): "investment_income_elected_form_4952" in summary["required_person_source_columns"] ) - assert "investment_interest_expense" in summary["required_person_source_columns"] + for column in l0_refit_export.US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + assert column in summary["required_person_source_columns"] assert ( "other_health_insurance_premiums" in summary["required_person_source_columns"] ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py b/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py index b7a53ca743..fa03998377 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_late_producer_dag.py @@ -345,7 +345,7 @@ def test_canonical_us_late_registry_has_exact_producer_surface() -> None: assert len(registry) == 38 assert len(groups) == 19 - assert sum(len(group.targets) for group in groups) == 70 + assert sum(len(group.targets) for group in groups) == 72 assert {contract.kind for contract in registry.values()} == { "primary_puf", "acs_earnings_universe", @@ -353,10 +353,10 @@ def test_canonical_us_late_registry_has_exact_producer_surface() -> None: "late_transfer", "source_finalizer", } - assert len(registry[US_LATE_PRIMARY_PUF_STAGE].inputs) == 119 + assert len(registry[US_LATE_PRIMARY_PUF_STAGE].inputs) == 121 primary_outputs = registry[US_LATE_PRIMARY_PUF_STAGE].outputs - assert len(primary_outputs) == 100 - assert sum(output.coverage_scope == "puf_clone" for output in primary_outputs) == 64 + assert len(primary_outputs) == 102 + assert sum(output.coverage_scope == "puf_clone" for output in primary_outputs) == 66 assert ( sum(output.coverage_scope == "whole_pool" for output in primary_outputs) == 35 ) @@ -434,11 +434,11 @@ def test_late_overlap_ownership_exhausts_every_permitted_dual_write() -> None: } - tail_owned declared = set(US_LATE_OVERLAP_OWNERSHIP_TARGETS) - assert len(primary) == 65 + assert len(primary) == 67 assert len(source_writes) == 35 assert len(source_writes & transfer) == 29 - assert len(transfer) == 70 - assert len(recipient_owned) == 60 + assert len(transfer) == 72 + assert len(recipient_owned) == 62 assert ( callback_passthroughs == { @@ -514,9 +514,7 @@ def test_late_overlap_ownership_exhausts_every_permitted_dual_write() -> None: ) assert owner_by_cell[(target, "acs", 0)] == transfer_producer_name( "person", - "puf_tax_itemization__batch_2" - if target == "traditional_ira_contributions_desired" - else "puf_tax_itemization__batch_3", + "puf_tax_itemization__batch_3", ) for clone_index in (1, 2): assert owner_by_cell[(target, "acs", clone_index)] == ( @@ -545,7 +543,7 @@ def test_primary_puf_inventory_declares_exact_read_before_write_surface() -> Non for requirement in US_LATE_PRIMARY_PUF_INPUT_INVENTORY.requirements } - assert len(requirements) == 114 + assert len(requirements) == 116 assert tuple( (item.entity, item.column, item.value_kind) for item in requirements["filing_status"].alternatives[0] @@ -626,7 +624,7 @@ def test_primary_puf_inventory_declares_exact_read_before_write_surface() -> Non def test_canonical_us_late_registry_declares_required_cross_producer_edges() -> None: edges = set(CANONICAL_US_LATE_PRODUCER_SCHEDULE.edges) - assert len(edges) == 71 + assert len(edges) == 70 assert ( US_LATE_ACS_EARNINGS_UNIVERSE_STAGE, US_LATE_PRIMARY_PUF_STAGE, @@ -808,7 +806,7 @@ def test_canonical_us_late_schedule_is_import_validated_and_byte_stable() -> Non assert receipt["producer_count"] == 38 assert receipt["source_producer_count"] == 16 assert receipt["transfer_group_count"] == 19 - assert receipt["transfer_target_count"] == 70 + assert receipt["transfer_target_count"] == 72 assert receipt["order"][:2] == [ US_LATE_ACS_EARNINGS_UNIVERSE_STAGE, US_LATE_PRIMARY_PUF_STAGE, @@ -1324,7 +1322,7 @@ def test_source_numeric_input_audit_is_fully_executable() -> None: } == {"finite_numeric"} -def test_late_target_dependency_kinds_partition_51_numeric_17_boolean_2_string() -> ( +def test_late_target_dependency_kinds_partition_53_numeric_17_boolean_2_string() -> ( None ): string_targets = {"ssn_card_type", "immigration_status_str"} @@ -1361,7 +1359,7 @@ def test_late_target_dependency_kinds_partition_51_numeric_17_boolean_2_string() } numeric_targets = set(observed) - boolean_targets - string_targets assert (len(numeric_targets), len(boolean_targets), len(string_targets)) == ( - 51, + 53, 17, 2, ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py index 4e15a813be..507ac09597 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool.py @@ -603,10 +603,10 @@ def test_pool_transfer_plan_extends_legacy_except_receipted_asset_deferrals() -> assert "has_marketplace_health_coverage" not in owners target_names = sorted(owners) - assert len(target_names) == 118 + assert len(target_names) == 120 assert ( hashlib.sha256(("\n".join(target_names) + "\n").encode()).hexdigest() - == "c792f12f0ef34f8a2ca9f16e68f5b306391eed56f120cda247bf778a95118c15" + == "80b7ebaf0fc8f4376ff86024d6b4b3ca6f870a6b65b271824fd515652f7b2bcc" ) @@ -626,10 +626,10 @@ def keys(target_families): source_producers = keys(pool_post_puf_source_producer_target_families()) assert len(early) == 48 - assert len(late) == 70 + assert len(late) == 72 assert early.isdisjoint(late) assert early | late == full - assert len(puf_producers) == 43 + assert len(puf_producers) == 45 assert len(source_producers) == 29 assert len(puf_producers & source_producers) == 2 assert puf_producers | source_producers == late @@ -661,15 +661,15 @@ def test_pool_input_surface_normalizes_all_four_source_registries() -> None: surface = pool_input_surface() by_name = {entry.variable: entry for entry in surface} - assert len(surface) == len(by_name) == 142 + assert len(surface) == len(by_name) == 144 assert [entry.variable for entry in surface] == sorted(by_name) assert Counter( provenance for entry in surface for provenance in entry.provenance ) == Counter( { - "pool_transfer_target_families": 118, + "pool_transfer_target_families": 120, "POOL_DEFERRED_TRANSFER_INPUTS": 3, - "PRIMARY_QRF_TARGET_ORDER": 65, + "PRIMARY_QRF_TARGET_ORDER": 67, "load_take_up_contract": 17, } ) @@ -1126,7 +1126,7 @@ def test_every_pool_transfer_target_is_an_installed_engine_input_leaf() -> None: for columns in families.values() for target in columns } - assert len(targets) == 118 + assert len(targets) == 120 acs_transfer_module.assert_acs_transfer_targets_are_input_leaves( targets, require_known=True, @@ -1151,9 +1151,9 @@ def test_every_pool_transfer_family_accepts_its_produced_physical_dtype( ) ) - assert len(targets) == 118 + assert len(targets) == 120 assert len(predictors) == 32 - assert len(primary_predictor_sets) == 65 + assert len(primary_predictor_sets) == 67 primary_targets = tuple( ( *puf_support_module.PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS, @@ -1166,9 +1166,9 @@ def test_every_pool_transfer_family_accepts_its_produced_physical_dtype( for position, target in enumerate(primary_targets) ) assert len(primary_predictor_sets[0][1]) == 8 - assert len(primary_predictor_sets[-1][1]) == 72 + assert len(primary_predictor_sets[-1][1]) == 74 assert len(POOL_DEFERRED_TRANSFER_INPUTS) == 3 - assert len(targets) + len(POOL_DEFERRED_TRANSFER_INPUTS) == 121 + assert len(targets) + len(POOL_DEFERRED_TRANSFER_INPUTS) == 123 assert set(POOL_SOURCE_OPERATOR_ORDER) <= set(calls) assert all(calls[name] > 0 for name in POOL_SOURCE_OPERATOR_ORDER) assert calls["with_us_prior_year_income_inputs"] == 2 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py index b329171528..d0a17f093a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_h5_io.py @@ -1108,7 +1108,7 @@ def _cached_canonical_stacked_late_dag_receipt( "completion": { "status": "complete", "group_count": 19, - "target_count": 70, + "target_count": 72, "residual_null_rows": 0, }, } diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_e19200_decomposition.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_e19200_decomposition.py index 0d4e1f7701..83dcad4d05 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_e19200_decomposition.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_e19200_decomposition.py @@ -1,15 +1,26 @@ from __future__ import annotations +import json +import math +from dataclasses import replace +from fractions import Fraction from hashlib import sha256 from pathlib import Path import numpy as np import pytest +from hypothesis import given, settings +from hypothesis import strategies as st +import microcosm.build.us_runtime.puf_interest_components as interest_module from microcosm.build.us_runtime.puf_interest_components import ( US_PUF_E19200_AGI_BANDS, US_PUF_E19200_ALL_RETURNS_COMPONENTS, + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, + puf_e19200_agi_bands_runtime_identity, + puf_e19200_interest_components_asset_identity, split_us_puf_e19200_by_agi_band, + split_us_puf_e19200_residual_by_agi_band, ) from microcosm.build.us_runtime.puf_source_agi import ( source_year_puf_adjusted_gross_income, @@ -34,6 +45,10 @@ _REAL_SOURCE_AGI_VECTOR_SHA256 = ( "8b2b6c206e8e9f7b80dcfd86554962b0966b25ad8c47eab7d9db4f27f69ebc0d" ) +_E19200_PERSON_OUTPUTS = ( + "home_mortgage_interest", + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, +) def _representative_agi(lower: float | None, upper: float | None) -> float: @@ -60,9 +75,12 @@ def _one_record_per_band_arrays() -> tuple[dict[str, list[object]], np.ndarray]: "filing_status": [b"SINGLE"] * len(tax_unit_ids), "person_tax_unit_id": tax_unit_ids.tolist(), "home_mortgage_interest": total_interest.tolist(), - # The processed PUF leaf is all-zero before this decomposition. The - # split must replace it from E19200, not preserve or add this sentinel. - "investment_interest_expense": np.zeros(len(tax_unit_ids)).tolist(), + # Processed PUF residual leaves are absent or all-zero before this + # decomposition. None may carry independently sourced values. + **{ + column: np.zeros(len(tax_unit_ids)).tolist() + for column in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS + }, } return arrays, adjusted_gross_income @@ -201,10 +219,7 @@ def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: donor = puf_tax_unit_donor_from_arrays( arrays, adjusted_gross_income=adjusted_gross_income, - person_outputs=( - "home_mortgage_interest", - "investment_interest_expense", - ), + person_outputs=_E19200_PERSON_OUTPUTS, tax_unit_outputs=(), ) @@ -218,16 +233,34 @@ def test_e19200_donor_split_preserves_each_band_and_published_shares() -> None: ) expected_non_mortgage = 1_000.0 - expected_mortgage np.testing.assert_allclose(donor["home_mortgage_interest"], expected_mortgage) - np.testing.assert_allclose( - donor["investment_interest_expense"], + expected_leaves = split_us_puf_e19200_residual_by_agi_band( expected_non_mortgage, + adjusted_gross_income, ) + for column, expected in zip( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, expected_leaves, strict=True + ): + np.testing.assert_allclose(donor[column], expected) np.testing.assert_array_equal( donor["home_mortgage_interest"].to_numpy() - + donor["investment_interest_expense"].to_numpy(), + + ( + ( + donor["home_mortgage_points"].to_numpy() + + donor["mortgage_insurance_premiums"].to_numpy() + ) + + donor["investment_interest_expense"].to_numpy() + ), np.full(len(total), 1_000.0), ) assert (donor["investment_interest_expense"] > 0).all() + original_mortgage, _ = split_us_puf_e19200_by_agi_band( + np.full(len(total), 1_000.0), + adjusted_gross_income, + ) + np.testing.assert_array_equal( + donor["home_mortgage_interest"].to_numpy().view(np.uint64), + original_mortgage.view(np.uint64), + ) # The same proportional rule applied to the literal source rows recovers # the published mortgage amounts and the full conserving residual mass. @@ -252,7 +285,7 @@ def test_e19200_donor_split_requires_explicit_adjusted_gross_income() -> None: "filing_status": [b"SINGLE"], "person_tax_unit_id": [1], "home_mortgage_interest": [100.0], - "investment_interest_expense": [0.0], + **{column: [0.0] for column in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS}, # This predictor is deliberately tempting but is not AGI and cannot # silently select a source-table band. "employment_income": [250_000.0], @@ -261,32 +294,50 @@ def test_e19200_donor_split_requires_explicit_adjusted_gross_income() -> None: with pytest.raises(ValueError, match="adjusted_gross_income"): puf_tax_unit_donor_from_arrays( arrays, - person_outputs=( - "home_mortgage_interest", - "investment_interest_expense", - ), + person_outputs=_E19200_PERSON_OUTPUTS, + tax_unit_outputs=(), + ) + + +@pytest.mark.parametrize("column", US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS) +def test_e19200_donor_split_rejects_independently_sourced_residual_leaves( + column: str, +) -> None: + arrays, adjusted_gross_income = _one_record_per_band_arrays() + arrays[column][0] = 1.0 + with pytest.raises(ValueError, match=f"nonzero {column}"): + puf_tax_unit_donor_from_arrays( + arrays, + adjusted_gross_income=adjusted_gross_income, + person_outputs=_E19200_PERSON_OUTPUTS, tax_unit_outputs=(), ) +def test_e19200_zero_donor_has_zero_leaves_without_agi() -> None: + arrays, _ = _one_record_per_band_arrays() + arrays["home_mortgage_interest"] = [0.0] * len(US_PUF_E19200_AGI_BANDS) + donor = puf_tax_unit_donor_from_arrays( + arrays, + person_outputs=_E19200_PERSON_OUTPUTS, + tax_unit_outputs=(), + ) + for column in _E19200_PERSON_OUTPUTS: + np.testing.assert_array_equal(donor[column], np.zeros(len(donor))) + + def test_e19200_donor_split_is_bit_deterministic_and_order_invariant() -> None: arrays, adjusted_gross_income = _one_record_per_band_arrays() first = puf_tax_unit_donor_from_arrays( arrays, adjusted_gross_income=adjusted_gross_income, - person_outputs=( - "home_mortgage_interest", - "investment_interest_expense", - ), + person_outputs=_E19200_PERSON_OUTPUTS, tax_unit_outputs=(), ) second = puf_tax_unit_donor_from_arrays( arrays, adjusted_gross_income=adjusted_gross_income, - person_outputs=( - "home_mortgage_interest", - "investment_interest_expense", - ), + person_outputs=_E19200_PERSON_OUTPUTS, tax_unit_outputs=(), ) @@ -298,15 +349,11 @@ def test_e19200_donor_split_is_bit_deterministic_and_order_invariant() -> None: shuffled = puf_tax_unit_donor_from_arrays( shuffled_arrays, adjusted_gross_income=adjusted_gross_income[permutation], - person_outputs=( - "home_mortgage_interest", - "investment_interest_expense", - ), + person_outputs=_E19200_PERSON_OUTPUTS, tax_unit_outputs=(), ).sort_values("tax_unit_id") - columns = ("home_mortgage_interest", "investment_interest_expense") - for column in columns: + for column in _E19200_PERSON_OUTPUTS: np.testing.assert_array_equal( first[column].to_numpy().view(np.uint64), second[column].to_numpy().view(np.uint64), @@ -427,6 +474,283 @@ def test_split_conserves_bit_exactly_on_adversarial_floats() -> None: assert (mortgage >= 0.0).all() and (non_mortgage >= 0.0).all() +@st.composite +def _agi_in_any_band(draw): + index = draw(st.integers(min_value=0, max_value=21)) + band = US_PUF_E19200_AGI_BANDS[index] + agi = draw( + st.floats( + min_value=band.lower_bound, + max_value=band.upper_bound, + exclude_max=band.upper_bound is not None, + allow_nan=False, + allow_infinity=False, + ) + ) + return index, agi + + +def _assert_three_leaf_conservation(total: float, agi: float, index: int) -> None: + band = US_PUF_E19200_AGI_BANDS[index] + totals = np.asarray([total], dtype=np.float64) + agis = np.asarray([agi], dtype=np.float64) + mortgage, residual = split_us_puf_e19200_by_agi_band(totals, agis) + # Freeze the pre-three-leaf arithmetic as an independent differential + # oracle: the existing mortgage component must keep every float64 bit. + historical_mortgage = totals * band.home_mortgage_share + historical_residual = totals - historical_mortgage + historical_mortgage = totals - historical_residual + points, premiums, investment = split_us_puf_e19200_residual_by_agi_band( + residual, agis + ) + np.testing.assert_array_equal( + mortgage.view(np.uint64), historical_mortgage.view(np.uint64) + ) + np.testing.assert_array_equal( + residual.view(np.uint64), historical_residual.view(np.uint64) + ) + leaves = [float(leaf[0]) for leaf in (mortgage, points, premiums, investment)] + assert all(math.isfinite(leaf) and leaf >= 0 for leaf in leaves) + # Fractions expose exact real arithmetic, independently of the grouping + # and any rounding in a floating-point sum. + assert sum(Fraction(leaf) for leaf in leaves) == Fraction(total) + assert sum(Fraction(leaf) for leaf in leaves[1:]) == Fraction(float(residual[0])) + assert (points[0] + premiums[0]) + investment[0] == residual[0] + assert mortgage[0] + ((points[0] + premiums[0]) + investment[0]) == total + reconstructed = np.asarray([math.fsum(leaves)], dtype=np.float64) + # math.fsum([-0.0, ...]) returns +0.0, as does the historical pair sum. + canonical_total = np.asarray([0.0 if total == 0.0 else total], dtype=np.float64) + np.testing.assert_array_equal( + reconstructed.view(np.uint64), canonical_total.view(np.uint64) + ) + if total == 0.0: + return + component_denominator = band.residual_component_amount + expected_shares = [ + float( + Fraction( + band.home_mortgage_interest_amount, band.total_interest_paid_amount + ) + ), + *[ + float( + Fraction( + band.non_mortgage_interest_amount * component, + band.total_interest_paid_amount * component_denominator, + ) + ) + for component in ( + band.deductible_points_amount, + band.qualified_mortgage_insurance_premiums_amount, + band.investment_interest_amount, + ) + ], + ] + # Complement reconciliation rounds at the scale of the original total, + # rather than the much smaller leaf. Subnormal totals additionally need + # an absolute minimum-float allowance; relative ULP bounds cannot hold + # when a positive published component rounds to zero. + share_tolerance = 4 * math.ulp(1.0) + 4 * (math.ulp(total) / total) + for leaf, expected_share in zip(leaves, expected_shares, strict=True): + assert abs(leaf / total - expected_share) <= share_tolerance + if band.qualified_mortgage_insurance_premiums_amount == 0: + assert premiums[0] == 0.0 + assert not np.signbit(premiums[0]) + + +@settings(max_examples=600, deadline=None) +@given( + total=st.floats( + min_value=0.0, + allow_nan=False, + allow_infinity=False, + allow_subnormal=True, + ), + indexed_agi=_agi_in_any_band(), +) +def test_three_leaf_split_exact_conservation_property( + total: float, indexed_agi: tuple[int, float] +) -> None: + index, agi = indexed_agi + _assert_three_leaf_conservation(total, agi, index) + + +@pytest.mark.parametrize( + "total", + [ + 0.0, + -0.0, + 1.53, + 0.01, + 0.1, + float(np.nextafter(0.0, 1.0)), + float(np.nextafter(0.0, 1.0) * 7), + float(np.nextafter(np.finfo(np.float64).tiny, 0.0)), + float(np.finfo(np.float64).tiny), + 1e-250, + 1e250, + float(np.finfo(np.float64).max), + ], +) +def test_three_leaf_split_adversarial_totals_in_every_band(total: float) -> None: + for index, band in enumerate(US_PUF_E19200_AGI_BANDS): + _assert_three_leaf_conservation( + total, _representative_agi(band.lower_bound, band.upper_bound), index + ) + + +def test_three_leaf_split_recovers_all_published_band_proportions() -> None: + bands = US_PUF_E19200_AGI_BANDS + agis = np.asarray( + [_representative_agi(band.lower_bound, band.upper_bound) for band in bands] + ) + totals = np.asarray( + [band.total_interest_paid_amount for band in bands], dtype=float + ) + mortgage, residual = split_us_puf_e19200_by_agi_band(totals, agis) + points, premiums, investment = split_us_puf_e19200_residual_by_agi_band( + residual, agis + ) + for index, band in enumerate(bands): + _assert_three_leaf_conservation(float(totals[index]), float(agis[index]), index) + scale = band.non_mortgage_interest_amount / band.residual_component_amount + np.testing.assert_allclose( + [points[index], premiums[index], investment[index]], + np.asarray( + [ + band.deductible_points_amount, + band.qualified_mortgage_insurance_premiums_amount, + band.investment_interest_amount, + ] + ) + * scale, + rtol=1e-12, + atol=4 * math.ulp(float(totals[index])), + ) + assert investment[-1] > 0.97 * residual[-1] + assert investment[-1] > mortgage[-1] + zero_premium = np.asarray( + [band.qualified_mortgage_insurance_premiums_amount == 0 for band in bands] + ) + assert zero_premium.any() + np.testing.assert_array_equal( + premiums[zero_premium].view(np.uint64), + np.zeros(int(zero_premium.sum())).view(np.uint64), + ) + + +def test_three_leaf_split_identity_binds_all_consumed_shares() -> None: + identity = puf_e19200_interest_components_asset_identity() + runtime_identity = puf_e19200_agi_bands_runtime_identity() + assert identity["agi_bands"] == runtime_identity["agi_bands"] + for band, resolved in zip( + US_PUF_E19200_AGI_BANDS, identity["agi_bands"], strict=True + ): + denominator = ( + band.deductible_points_amount + + band.qualified_mortgage_insurance_premiums_amount + + band.investment_interest_amount + ) + assert resolved["deductible_points_residual_share"] == ( + band.deductible_points_amount / denominator + ) + assert resolved["qualified_mortgage_insurance_premiums_residual_share"] == ( + band.qualified_mortgage_insurance_premiums_amount / denominator + ) + assert resolved["investment_interest_residual_share"] == ( + band.investment_interest_amount / denominator + ) + assert resolved["deductible_points_within_points_and_premiums_share"] == ( + band.deductible_points_amount + / ( + band.deductible_points_amount + + band.qualified_mortgage_insurance_premiums_amount + ) + ) + first = US_PUF_E19200_AGI_BANDS[0] + changed = replace( + first, + deductible_points_amount=first.deductible_points_amount + 1, + qualified_mortgage_insurance_premiums_amount=( + first.qualified_mortgage_insurance_premiums_amount - 1 + ), + ) + changed_identity = puf_e19200_agi_bands_runtime_identity( + (changed, *US_PUF_E19200_AGI_BANDS[1:]) + ) + assert changed.home_mortgage_share == first.home_mortgage_share + assert ( + changed.investment_interest_residual_share + == first.investment_interest_residual_share + ) + assert changed_identity["sha256"] != runtime_identity["sha256"] + + +def test_three_leaf_source_loader_rejects_zero_points_and_premiums( + tmp_path: Path, +) -> None: + source = interest_module.files("microcosm.build.us").joinpath( + interest_module._SOURCE_ASSET + ) + payload = json.loads(source.read_text(encoding="utf-8")) + first = payload["agi_bands"][0] + first["investment_interest_amount"] += ( + first["deductible_points_amount"] + + first["qualified_mortgage_insurance_premiums_amount"] + ) + first["deductible_points_amount"] = 0 + first["qualified_mortgage_insurance_premiums_amount"] = 0 + invalid_asset = tmp_path / interest_module._SOURCE_ASSET + invalid_asset.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(ValueError, match=r"row 11 has zero CN \+ CP"): + puf_e19200_interest_components_asset_identity(invalid_asset) + + +def test_three_leaf_split_zero_denominator_has_no_points_or_premiums( + monkeypatch: pytest.MonkeyPatch, +) -> None: + absent = replace( + US_PUF_E19200_AGI_BANDS[0], + deductible_points_amount=0, + qualified_mortgage_insurance_premiums_amount=0, + ) + assert absent.deductible_points_within_points_and_premiums_share == 0.0 + for name, value in ( + ("_POINTS_AND_PREMIUMS_AMOUNTS", 0), + ("_INVESTMENT_RESIDUAL_SHARES", 1.0), + ("_POINTS_WITHIN_POINTS_AND_PREMIUMS_SHARES", 0.0), + ): + values = getattr(interest_module, name).copy() + values[0] = value + monkeypatch.setattr(interest_module, name, values) + points, premiums, investment = split_us_puf_e19200_residual_by_agi_band( + [1.53], [0.0] + ) + np.testing.assert_array_equal(points, [0.0]) + np.testing.assert_array_equal(premiums, [0.0]) + np.testing.assert_array_equal(investment, [1.53]) + + +@pytest.mark.parametrize( + ("residual", "agi", "message"), + [ + ([[1.0]], [0.0], "one-dimensional"), + ([1.0], [[0.0]], "one-dimensional"), + ([1.0], [], "same record count"), + ([float("nan")], [0.0], "finite"), + ([float("inf")], [0.0], "finite"), + ([1.0], [float("nan")], "finite"), + ([1.0], [float("inf")], "finite"), + ([-1.0], [0.0], "nonnegative"), + ], +) +def test_three_leaf_split_rejects_invalid_inputs( + residual: object, agi: object, message: str +) -> None: + with pytest.raises(ValueError, match=message): + split_us_puf_e19200_residual_by_agi_band(residual, agi) + + def test_donor_api_rejects_non_real_agi_through_a_valid_fixture() -> None: """PR #561 review findings 2 (rounds 1-2): the donor API must fail closed on AGI that is not a finite real number. Round 1's diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py index 17e1ff3a35..a997742585 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support.py @@ -1,6 +1,8 @@ """Tests split from packages/microcosm-build/tests/test_us_puf_support.py.""" # ruff: noqa: F403, F405 +import math + from test_support.microcosm_build.us_puf_support import * @@ -255,6 +257,8 @@ def test_puf_tax_unit_donor_quarantines_only_mortgage_fields() -> None: adjusted_gross_income=[0.0, 0.0, 10_000_000.0], person_outputs=( "home_mortgage_interest", + "home_mortgage_points", + "mortgage_insurance_premiums", "investment_interest_expense", "employment_income_before_lsr", "short_term_capital_gains", @@ -290,13 +294,19 @@ def test_puf_tax_unit_donor_quarantines_only_mortgage_fields() -> None: ] raw_total = np.asarray([10_000_000.0, 5_000_000.0, 10_500_000.0]) - mortgage, investment = split_us_puf_e19200_by_agi_band( + mortgage, non_mortgage = split_us_puf_e19200_by_agi_band( raw_total, np.asarray([0.0, 0.0, 10_000_000.0]), ) + points, premiums, investment = split_us_puf_e19200_residual_by_agi_band( + non_mortgage, + np.asarray([0.0, 0.0, 10_000_000.0]), + ) share = mortgage / raw_total expected_before_quarantine = { "home_mortgage_interest": mortgage, + "home_mortgage_points": points, + "mortgage_insurance_premiums": premiums, "investment_interest_expense": investment, "first_home_mortgage_interest": ( np.asarray([8_000_000.0, 4_000_000.0, 7_000_000.0]) * share @@ -347,7 +357,7 @@ def test_puf_tax_unit_donor_quarantines_only_mortgage_fields() -> None: field = quarantine["fields"][column] screened = expected[[0, 2]] assert field["screened_record_count"] == 2 - assert field["screened_nonzero_record_count"] == 2 + assert field["screened_nonzero_record_count"] == np.count_nonzero(screened) assert field["screened_weight"] == 404.0 assert field["screened_unweighted_signed_mass"] == pytest.approx(screened.sum()) assert field["screened_weighted_signed_mass"] == pytest.approx( @@ -363,6 +373,8 @@ def test_puf_tax_unit_donor_quarantines_only_mortgage_fields() -> None: assert puf_support_module.US_PUF_DONOR_MORTGAGE_QUARANTINE_FIELDS == ( "home_mortgage_interest", + "home_mortgage_points", + "mortgage_insurance_premiums", "investment_interest_expense", "first_home_mortgage_interest", "second_home_mortgage_interest", @@ -392,7 +404,9 @@ def test_puf_tax_unit_donor_rejects_reserved_screen_column_output() -> None: ) -def test_puf_e19200_split_scales_only_lineage_and_populates_residual() -> None: +def test_puf_e19200_split_scales_only_lineage_and_populates_three_residual_leaves() -> ( + None +): # Pin the lineage tuple by exact membership: an accidental addition (the # sol round-1 failure mode was appending investment_interest_expense, # invisible behind a zero sentinel) must fail here, not silently carve a @@ -431,6 +445,8 @@ def test_puf_e19200_split_scales_only_lineage_and_populates_residual() -> None: "second_home_mortgage_balance": [0.0, 125_000.0], "first_home_mortgage_origination_year": [2018.0, 2016.0], "second_home_mortgage_origination_year": [0.0, 2020.0], + "home_mortgage_points": [0.0, 0.0], + "mortgage_insurance_premiums": [0.0, 0.0], "investment_interest_expense": [0.0, 0.0], puf_support_module._MORTGAGE_OUTLIER_SCREEN_COLUMN: [100.0, 200.0], puf_support_module._E19200_AGI_BAND_COLUMN: [0.0, 10_000_000.0], @@ -450,7 +466,16 @@ def test_puf_e19200_split_scales_only_lineage_and_populates_residual() -> None: donor[column].to_numpy(), original[column].to_numpy() * shares, ) - np.testing.assert_allclose(donor["investment_interest_expense"], non_mortgage) + residual_leaves = split_us_puf_e19200_residual_by_agi_band( + non_mortgage, + np.asarray([0.0, 10_000_000.0]), + ) + for column, values in zip( + puf_support_module.US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, + residual_leaves, + strict=True, + ): + np.testing.assert_array_equal(donor[column], values) for column in ( "real_estate_taxes", "first_home_mortgage_balance", @@ -463,6 +488,44 @@ def test_puf_e19200_split_scales_only_lineage_and_populates_residual() -> None: assert puf_support_module._E19200_AGI_BAND_COLUMN not in donor +def test_puf_e19200_donor_preserves_splitter_mortgage_bits_and_exact_leaf_sum() -> None: + totals = np.asarray( + [0.0, 1.53, 659_340.46, np.nextafter(0.0, 1.0), np.finfo(np.float64).max] + ) + agi = np.asarray([0.0, 0.0, 10_000_000.0, 50_000.0, 10_000_000.0]) + mortgage, _residual = split_us_puf_e19200_by_agi_band(totals, agi) + reconstructed = totals * np.divide( + mortgage, totals, out=np.ones_like(totals), where=totals != 0 + ) + assert np.any(reconstructed.view(np.uint64) != mortgage.view(np.uint64)) + donor = pd.DataFrame( + { + "home_mortgage_interest": totals, + **{ + column: np.zeros_like(totals) + for column in puf_support_module.US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS + }, + puf_support_module._MORTGAGE_OUTLIER_SCREEN_COLUMN: totals, + puf_support_module._E19200_AGI_BAND_COLUMN: agi, + } + ) + + puf_support_module._split_us_puf_e19200_components(donor) + + np.testing.assert_array_equal( + donor["home_mortgage_interest"].to_numpy().view(np.uint64), + mortgage.view(np.uint64), + ) + leaves = donor[ + [ + "home_mortgage_interest", + *puf_support_module.US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, + ] + ].to_numpy() + summed = np.asarray([math.fsum(row) for row in leaves]) + np.testing.assert_array_equal(summed.view(np.uint64), totals.view(np.uint64)) + + def test_puf_tax_detail_default_person_outputs_are_engine_leaves() -> None: assert "qualified_dividend_income" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "non_qualified_dividend_income" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS @@ -478,13 +541,11 @@ def test_puf_tax_detail_default_person_outputs_are_engine_leaves() -> None: assert "self_employment_income_before_lsr" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "self_employment_income" not in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert "tax_exempt_interest_income" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS - assert "investment_interest_expense" in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS - assert ( - "investment_interest_expense" - in puf_support_module._PUF_TAX_DETAIL_NONNEGATIVE_OUTPUTS - ) - assert "investment_interest_expense" not in PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS - assert "investment_interest_expense" not in PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS + for column in puf_support_module.US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + assert column in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS + assert column in puf_support_module._PUF_TAX_DETAIL_NONNEGATIVE_OUTPUTS + assert column not in PUF_TAX_DETAIL_DEFAULT_TAX_UNIT_OUTPUTS + assert column not in PUF_TAX_DETAIL_FORMULA_OWNED_OUTPUTS assert "unemployment_compensation" not in PUF_TAX_DETAIL_DEFAULT_PERSON_OUTPUTS assert ( "long_term_capital_gains_before_response" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py b/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py index 5b1c1cc913..bbcbb47d2a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_release_head_to_head_scorer.py @@ -180,13 +180,13 @@ def test_canonical_battery_contract_matches_production_registries() -> None: single = len(module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY) joint = len(module.CANONICAL_ORIGIN_BATTERY_JOINT_METRIC_REGISTRY) - assert single == 134 + assert single == 136 assert joint == 1 assert len(contract) == single + joint - assert sum(len(row["metric_legs"]) for row in contract.values()) == 372 + assert sum(len(row["metric_legs"]) for row in contract.values()) == 380 assert ( sum(row["metric"] == "monetary_sign_separated" for row in contract.values()) - == 79 + == 81 ) assert sum(row["metric"] == "boolean_incidence" for row in contract.values()) == 51 assert sum(row["metric"] == "categorical_tvd" for row in contract.values()) == 5 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py index 789b149a05..a9c755418f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py @@ -71,15 +71,15 @@ def test_constant_derived_domain_counts_are_complete( } assert target_counts == { "gap_fill_stacked_spine": 48, - "primary_puf_qrf": 65, - "late_producer_dag": 70, + "primary_puf_qrf": 67, + "late_producer_dag": 72, } assert "primary_effective_predictor_tuples" not in imputation["chaining"] - assert len(derive_primary_effective_predictor_tuples(imputation)) == 65 + assert len(derive_primary_effective_predictor_tuples(imputation)) == 67 itemization_batches = [ family for family in families if "puf_tax_itemization__batch_" in family["id"] ] - assert [len(family["targets"]) for family in itemization_batches] == [8, 8, 8, 8, 5] + assert [len(family["targets"]) for family in itemization_batches] == [8, 8, 8, 8, 7] modeled_output_scopes = { stage: Counter( target["output_coverage_scope"] @@ -90,8 +90,8 @@ def test_constant_derived_domain_counts_are_complete( for stage in ("primary_puf_qrf", "late_producer_dag") } assert modeled_output_scopes == { - "primary_puf_qrf": {"puf_clone": 64, "whole_pool": 1}, - "late_producer_dag": {"whole_pool": 70}, + "primary_puf_qrf": {"puf_clone": 66, "whole_pool": 1}, + "late_producer_dag": {"whole_pool": 72}, } assert all( "output_coverage_scope" not in target @@ -138,15 +138,15 @@ def test_constant_derived_domain_counts_are_complete( spine_document=spine, bundle_document=bundle, )["late_producer_schedule_receipt"] - assert len(compiled_schedule["edges"]) == 71 - assert len(compiled_schedule["waves"]) == 6 + assert len(compiled_schedule["edges"]) == 70 + assert len(compiled_schedule["waves"]) == 5 assert ( compiled_schedule["schedule_sha256"] - == "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f" + == "4e5e538d9f7021d7649f06bf7b28c7374ed74b4da242ccd24dd8ccce284bd116" ) assert ( compiled_schedule["payload_sha256"] - == "7be038d34f228d66c12b53558fc5f30c93f1b376f1058c5e4fd7e7563a88d67f" + == "5a33b949600aaa016b1a2a36efbe0fefd3f4072900f793d030a2e1d1b4b44073" ) assert len(take_up["programs"]) == 17 @@ -199,7 +199,7 @@ def test_constant_derived_domain_counts_are_complete( for step in local_steps ) assert all(step["kernel"].startswith("kernel:") for step in take_up_steps) - assert len(battery["metric_registry"]) == 134 + assert len(battery["metric_registry"]) == 136 assert len(battery["joint_metric_registry"]) == 1 assert "metric_counts" not in battery assert "declared_surface" not in battery @@ -208,7 +208,7 @@ def test_constant_derived_domain_counts_are_complete( assert battery_views["metric_counts"] == { "boolean_incidence": 51, "categorical_tvd": 4, - "monetary_sign_separated": 79, + "monetary_sign_separated": 81, } assert set(calibration["targets"]) == { "cd_policy", @@ -232,8 +232,8 @@ def test_constant_derived_domain_counts_are_complete( for knob in ("k", "pi_hi", "seed"): assert selection["exact_k"][knob]["required"] is True assert selection["exact_k"][knob]["default"] is None - assert len(catalogs["columns"]) == 176 - assert len(resolved_us_spec.columns) == 176 + assert len(catalogs["columns"]) == 178 + assert len(resolved_us_spec.columns) == 178 assert Counter(artifact.kind for artifact in resolved_us_spec.artifacts) == { "producer_node": 38, "virtual_output": 18, @@ -242,6 +242,87 @@ def test_constant_derived_domain_counts_are_complete( assert len(resolved_us_spec.scopes) == 7 +def test_e19200_residual_leaves_have_closed_source_and_person_contracts( + resolved_us_spec: ResolvedSpec, +) -> None: + from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, + ) + + sources = _domain(resolved_us_spec, ResourceKind.SOURCES) + puf = next( + stage for stage in sources["stages"] if stage["stage"] == "puf_tax_detail" + ) + split = next( + operation + for operation in puf["operations"] + if operation.get("method") == "decompose_e19200_by_soi_agi_band" + ) + assert ( + split["points_output"], + split["premiums_output"], + split["investment_output"], + ) == US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS + assert ( + split["published_deductible_points_column"], + split["published_mortgage_insurance_premiums_column"], + split["published_investment_interest_column"], + ) == ("CN", "CP", "CR") + assert "non_mortgage_output" not in split + assert set(US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS) <= set(puf["outputs"]) + assert set(US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS) <= set(puf["nonnegative_outputs"]) + + catalogs = { + row["key"]: row["contract"] + for row in _domain(resolved_us_spec, ResourceKind.CATALOGS)["columns"] + } + imputation = _domain(resolved_us_spec, ResourceKind.IMPUTATION) + primary = next( + family + for family in imputation["families"] + if family["stage"] == "primary_puf_qrf" + ) + targets = {row["name"]: row for row in primary["targets"]} + for leaf in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS: + assert catalogs[f"person.{leaf}"]["dtype"] == "float64" + assert catalogs[f"person.{leaf}"]["definition_period"] == "year" + assert targets[leaf]["entity"] == "person" + assert targets[leaf]["output_coverage_scope"] == "puf_clone" + + +@pytest.mark.parametrize( + "leaf", ["home_mortgage_points", "mortgage_insurance_premiums"] +) +def test_bundle_generator_pending_inputs_fail_closed_after_engine_release( + leaf: str, +) -> None: + from microcosm.frame.schema import VariableMetadata + from tools.us_bundle_generation.pending_engine_inputs import ( + generator_variable_metadata, + ) + + class MetadataIndex: + def __init__(self, entity: str | None) -> None: + self.entity = entity + + def variable_metadata(self, name: str) -> VariableMetadata: + if self.entity is None: + raise ValueError(f"Unknown PolicyEngine-US source variable {name!r}.") + return VariableMetadata( + name=name, entity=self.entity, dtype="float", period="year" + ) + + expected = VariableMetadata( + name=leaf, entity="person", dtype="float", period="year" + ) + assert generator_variable_metadata(MetadataIndex(None), leaf) == expected + assert generator_variable_metadata(MetadataIndex("person"), leaf) == expected + with pytest.raises(ValueError, match="differs from its reviewed"): + generator_variable_metadata(MetadataIndex("tax_unit"), leaf) + with pytest.raises(ValueError, match="Unknown PolicyEngine-US"): + generator_variable_metadata(MetadataIndex(None), "unknown_input") + + def test_legacy_seed_vintage_and_publication_grammars_are_pinned( resolved_us_spec: ResolvedSpec, ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py index 773b3b06e4..6a19fe79bd 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_stacked_spine.py @@ -1205,7 +1205,7 @@ def test_canonical_authority_objects_are_deeply_immutable() -> None: calibration["scope"]["reference"] = "forged" -def test_canonical_metric_registry_covers_the_declared_134_target_split() -> None: +def test_canonical_metric_registry_covers_the_declared_136_target_split() -> None: surface = stacked_spine_module.CANONICAL_STACKED_DECLARED_SURFACE registry = stacked_spine_module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY surface_targets = { @@ -1215,9 +1215,9 @@ def test_canonical_metric_registry_covers_the_declared_134_target_split() -> Non for target in targets } - assert len(surface_targets) == 134 + assert len(surface_targets) == 136 assert Counter(entity for entity, _family, _target, _clone in surface_targets) == { - "person": 114, + "person": 116, "tax_unit": 12, "spm_unit": 8, } @@ -1233,7 +1233,7 @@ def test_canonical_metric_registry_covers_the_declared_134_target_split() -> Non ) assert set(registry) == surface_targets assert Counter(registry.values()) == { - "monetary_sign_separated": 79, + "monetary_sign_separated": 81, "boolean_incidence": 51, "categorical_tvd": 4, } @@ -1274,13 +1274,13 @@ def test_canonical_metric_registry_covers_the_declared_134_target_split() -> Non for target in targets } assert len(gap_targets) == 48 - assert len(post_puf_targets) == 70 - assert len(puf_producer_targets) == 43 + assert len(post_puf_targets) == 72 + assert len(puf_producer_targets) == 45 assert len(source_producer_targets) == 29 assert len(puf_producer_targets & source_producer_targets) == 2 assert puf_producer_targets | source_producer_targets == post_puf_targets assert gap_targets.isdisjoint(post_puf_targets) - assert len(gap_targets | post_puf_targets) == 118 + assert len(gap_targets | post_puf_targets) == 120 assert gap_targets | post_puf_targets < surface_targets assert not { "bank_account_assets", @@ -1304,7 +1304,7 @@ def test_canonical_metric_registry_drives_checkpoint_round_trip( registry = stacked_spine_module.CANONICAL_ORIGIN_BATTERY_METRIC_REGISTRY nullable_booleans = _transferred_registry_boolean_targets() assert Counter(registry.values()) == { - "monetary_sign_separated": 79, + "monetary_sign_separated": 81, "boolean_incidence": 51, "categorical_tvd": 4, } @@ -1472,7 +1472,7 @@ def test_registry_drives_every_late_callback_dtype_family_check() -> None: (entity, column): metric for (entity, _family, column, _clone_index), metric in registry.items() } - assert len(by_column) == len(registry) == 134 + assert len(by_column) == len(registry) == 136 representative = { "monetary_sign_separated": pd.Series([1.0, pd.NA], dtype="Float64"), @@ -1514,11 +1514,11 @@ def test_registry_drives_every_late_callback_dtype_family_check() -> None: for output in contract.outputs if (key := (output.entity, output.column)) in by_column ] - assert len(registered_occurrences) == 163 + assert len(registered_occurrences) == 167 assert Counter( metric for _producer, _entity, _column, metric in registered_occurrences ) == { - "monetary_sign_separated": 120, + "monetary_sign_separated": 124, "boolean_incidence": 37, "categorical_tvd": 6, } @@ -1526,9 +1526,9 @@ def test_registry_drives_every_late_callback_dtype_family_check() -> None: (entity, column): metric for _producer, entity, column, metric in registered_occurrences } - assert len(unique_late_targets) == 90 + assert len(unique_late_targets) == 92 assert Counter(unique_late_targets.values()) == { - "monetary_sign_separated": 67, + "monetary_sign_separated": 69, "boolean_incidence": 20, "categorical_tvd": 3, } @@ -2632,11 +2632,27 @@ def test_gap_fill_rejects_cloned_frames() -> None: _gap_fill_with_test_authority(cloned, plan=_GAP_FILL_TEST_PLAN, seed=578) -def test_bounded_transfer_group_remaps_canonical_producer_roles() -> None: +@pytest.mark.parametrize( + ("batch", "source_targets"), + [ + (2, ()), + ( + 3, + ( + "traditional_ira_contributions_desired", + "self_employed_pension_contributions_desired", + ), + ), + ], +) +def test_bounded_transfer_group_remaps_canonical_producer_roles( + batch: int, + source_targets: tuple[str, ...], +) -> None: group = next( group for group in stacked_spine_module.CANONICAL_US_LATE_TRANSFER_GROUPS - if group.family == "puf_tax_itemization__batch_2" + if group.family == f"puf_tax_itemization__batch_{batch}" ) puf_roles = stacked_spine_module._producer_role_surface_for_group( @@ -2648,13 +2664,17 @@ def test_bounded_transfer_group_remaps_canonical_producer_roles() -> None: stacked_spine_module.CANONICAL_STACKED_POST_PUF_SOURCE_PRODUCER_SURFACE, ) - assert "qualified_tuition_expenses" in puf_roles["person"][group.family] - assert "traditional_ira_contributions_desired" in puf_roles["person"][group.family] - assert source_roles == { - "person": { - group.family: ("traditional_ira_contributions_desired",), - } - } + if batch == 2: + assert { + "qualified_tuition_expenses", + "home_mortgage_points", + "mortgage_insurance_premiums", + } <= set(puf_roles["person"][group.family]) + else: + assert set(source_targets) <= set(puf_roles["person"][group.family]) + assert source_roles == ( + {"person": {group.family: source_targets}} if source_targets else {} + ) def test_late_readiness_rejects_object_typed_nonfinite_numeric_input() -> None: @@ -7773,7 +7793,7 @@ def test_completeness_receipts_bind_live_authority_per_target() -> None: ) canonical = stacked_completeness_gate(frame) assert canonical.passed, canonical.failures - assert canonical.details["declared_targets"] == 134 + assert canonical.details["declared_targets"] == 136 authority = canonical.details["authority"] assert authority["authority_form"] == "CANONICAL" assert authority["canonical"] is True @@ -8261,7 +8281,7 @@ def test_fresh_gate_result_cannot_graft_canonical_authority_onto_test_surface() with pytest.raises( ValueError, - match="must declare exactly 134 targets.*manifest emission is forbidden", + match="must declare exactly 136 targets.*manifest emission is forbidden", ): GateReport((grafted,)).to_manifest() @@ -8396,7 +8416,7 @@ def test_fresh_battery_result_cannot_forge_canonical_coverage_receipts() -> None with pytest.raises( ValueError, - match="coverage receipt must bind all 134 targets.*emission is forbidden", + match="coverage receipt must bind all 136 targets.*emission is forbidden", ): GateReport((forged,)).to_manifest() @@ -8810,7 +8830,7 @@ def test_battery_taxable_interest_metric_cannot_be_relabelled_rare_incidence() - assert not result.passed assert result.details["tested_comparisons"] == 0 metric_receipt = result.details["authority"]["components"]["metric_registry"] - assert metric_receipt["target_count"] == 134 + assert metric_receipt["target_count"] == 136 assert any( "person/puf_tax_itemization/taxable_interest_income[clone_0]" in failure and "authoritative metric 'monetary_sign_separated'" in failure diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index ca014679be..06e6bfc268 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -106,7 +106,7 @@ def test_stacked_tool_entrypoint_fixture_e2e_emits_one_logbook_row_at_every_term assert published_transfer["completion"] == { "status": "complete", "group_count": 19, - "target_count": 70, + "target_count": 72, "residual_null_rows": 0, } calibrated = [ @@ -925,7 +925,7 @@ def identity() -> dict[str, object]: assert set(resource_rows[producer]) == ( stacked_spine_module._late_contract_available_input_keys(contract) ) - assert pool_code["primary_qrf_checkpoint_schema_version"] == 6 + assert pool_code["primary_qrf_checkpoint_schema_version"] == 7 assert pool_code["puf_capital_gains_tail_manifest_schema_version"] == 2 assert pool_code["puf_capital_gains_tail_support_contract"] == ( pool_tool.puf_capital_gains_tail_support_contract_identity() diff --git a/packages/microcosm-frame/tests/golden/concept-coverage/policyengine-us.json b/packages/microcosm-frame/tests/golden/concept-coverage/policyengine-us.json index 01a6c79c5d..19dd3d1ef4 100644 --- a/packages/microcosm-frame/tests/golden/concept-coverage/policyengine-us.json +++ b/packages/microcosm-frame/tests/golden/concept-coverage/policyengine-us.json @@ -546,7 +546,7 @@ ], "engine": "policyengine-us", "engine_version": "2.2.1", - "input_count": 926, + "input_count": 928, "schema_sha256": "d824f5e97d58292d62f65ffae60bc8654408711344b638d4e3caa95d6c2e8d1e", "structural_inputs": [ { @@ -1767,6 +1767,10 @@ "entity": "person", "name": "home_is_on_agricultural_land" }, + { + "entity": "person", + "name": "home_mortgage_points" + }, { "entity": "spm_unit", "name": "homeowners_association_fees" @@ -2663,6 +2667,10 @@ "entity": "spm_unit", "name": "months_since_calworks_exit" }, + { + "entity": "person", + "name": "mortgage_insurance_premiums" + }, { "entity": "person", "name": "ms_ccpp_provider_type" diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 1d051f5c4d..05df2f5917 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -110,20 +110,20 @@ def binding() -> dict[str, object]: "early_families": 13, "early_targets": 48, "itemization_batches": 5, - "itemization_targets": 37, + "itemization_targets": 39, "late_groups": 19, - "late_targets": 70, + "late_targets": 72, "ownership_rows": 18, - "primary_effective_predictor_tuples": 65, + "primary_effective_predictor_tuples": 67, "primary_families": 1, - "primary_targets": 65, + "primary_targets": 67, "producer_authored_outputs": 92, - "producer_compiled_outputs": 227, - "producer_inputs": 2_744, + "producer_compiled_outputs": 231, + "producer_inputs": 2_748, "producer_nodes": 38, "producer_virtual_resources": 75, "release_rungs": 5, - "resolved_references": 334, + "resolved_references": 336, "seed_owner_bindings": 112, "seed_owner_rows": 54, "seed_sites": 53, @@ -133,11 +133,11 @@ def binding() -> dict[str, object]: "stacked_checkpoint_full_components": 13, "stacked_checkpoint_pool_code_components": 19, "stacked_checkpoint_static_components": 10, - "tail_control_fields": 934, + "tail_control_fields": 1112, "take_up_pipeline_steps": 28, "take_up_programs": 17, "typed_artifacts": 84, - "typed_columns": 176, + "typed_columns": 178, "typed_entities": 8, "typed_scopes": 7, } diff --git a/test_support/microcosm_build/us_multispine_pool.py b/test_support/microcosm_build/us_multispine_pool.py index 47468ad9a6..c7dc46bb83 100644 --- a/test_support/microcosm_build/us_multispine_pool.py +++ b/test_support/microcosm_build/us_multispine_pool.py @@ -960,7 +960,7 @@ def _assert_pool_transfer_produced_encodings( # (microcosm#982). assert primary_predictors == puf_support_module.PUF_TAX_DETAIL_DEFAULT_PREDICTORS assert len(primary_predictors) == 8 - assert len(primary_targets) == 65 + assert len(primary_targets) == 67 primary_qrf_observations = [ observation diff --git a/test_support/microcosm_build/us_multispine_pool_tool.py b/test_support/microcosm_build/us_multispine_pool_tool.py index 4117d5021a..8173dab4e1 100644 --- a/test_support/microcosm_build/us_multispine_pool_tool.py +++ b/test_support/microcosm_build/us_multispine_pool_tool.py @@ -1342,7 +1342,7 @@ def _canonical_late_transfer_receipt( "completion": { "status": "complete", "group_count": 19, - "target_count": 70, + "target_count": 72, "residual_null_rows": 0, }, } diff --git a/test_support/microcosm_build/us_puf_support.py b/test_support/microcosm_build/us_puf_support.py index 502744f568..84b4ed4c1b 100644 --- a/test_support/microcosm_build/us_puf_support.py +++ b/test_support/microcosm_build/us_puf_support.py @@ -22,6 +22,7 @@ impute_us_puf_tax_detail_support, puf_tax_unit_donor_from_arrays, split_us_puf_e19200_by_agi_band, + split_us_puf_e19200_residual_by_agi_band, support_channel_column, support_clone_index_column, support_source_id_column, diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index ef4bc8868b..5b705561f7 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "9e807e4bb83ded40c177c130515faad2fdcde747c6acb825e0d601e1609538b7" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index be6f9f56f2..a3ef16b40f 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -42,6 +42,9 @@ import json from pathlib import Path +from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, +) from microcosm.build.us_runtime.release_input_coverage import ( REFERENCE_ECPS_LAYER_RENAMES, project_ecps_parity_known_gap_names, @@ -75,6 +78,7 @@ # Schedule-D leg. These later inputs are hard requirements because the # shipped validation provisions must bind. POST_REFERENCE_ECPS_REQUIRED_INPUTS = ( + *US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS[:2], "fsla_overtime_premium", "qualified_passenger_vehicle_loan_interest", "traditional_401k_contributions_desired", @@ -113,6 +117,21 @@ # from today's artifacts is the intended red gate (the #368 SSI-asset # pattern, extended by #462 to the capital-gain-distributions route). POST_REFERENCE_COLUMN_NOTES = { + "home_mortgage_points": ( + "Deductible points not already carried in home_mortgage_interest " + "(Schedule A line 8c), derived from PUF E19200 using published SOI " + "Table 2.1 column CN within each AGI band's non-mortgage residual. " + "No frozen eCPS nonzero-share reference exists. Required without " + "a reviewed exclusion; pending the PolicyEngine-US leaf-input release." + ), + "mortgage_insurance_premiums": ( + "Qualified mortgage-insurance premiums (Schedule A line 8d), " + "derived from PUF E19200 using published SOI Table 2.1 column CP " + "within each AGI band's non-mortgage residual, before any engine " + "AGI phase-out. No frozen eCPS nonzero-share reference exists. " + "Required without a reviewed exclusion; pending the PolicyEngine-US " + "leaf-input release." + ), "schedule_d_capital_gain_distributions": ( "Schedule D line 13 route leg of the #282 capital-gain-distributions " "split (memo component of long_term_capital_gains, written by the " @@ -1616,6 +1635,8 @@ def build_manifest() -> dict: "Required surface = input columns in the pinned, sha-verified " "ecps_parity_reference.json populated layers, plus the documented " "post-reference fsla_overtime_premium, " + "home_mortgage_points and mortgage_insurance_premiums from the " + "source-backed E19200 decomposition (no frozen eCPS reference), " "qualified_passenger_vehicle_loan_interest, five desired " "retirement-contribution inputs, " "meets_ssi_disability_criteria required by shipped validation " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 5dc45835db..235f823204 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "9e807e4bb83ded40c177c130515faad2fdcde747c6acb825e0d601e1609538b7" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" @@ -348,7 +348,7 @@ def assert_frozen_legacy_compatibility( ) -def rendered_files() -> dict[Path, bytes]: +def rendered_files(*, preserve_engine_lock: bool = False) -> dict[Path, bytes]: documents = build_documents() assert_frozen_legacy_compatibility(documents) files = { @@ -362,21 +362,24 @@ def rendered_files() -> dict[Path, bytes]: + b"\n" ) files[US_PACKAGE_ROOT / "country_package.json"] = manifest - files[US_PACKAGE_ROOT / ENGINE_ABI_LOCK_FILENAME] = ( - engine_abi_lock_bytes_from_domains( - { - filename.removesuffix(".yaml"): document - for filename, document in documents.items() - } + if not preserve_engine_lock: + files[US_PACKAGE_ROOT / ENGINE_ABI_LOCK_FILENAME] = ( + engine_abi_lock_bytes_from_domains( + { + filename.removesuffix(".yaml"): document + for filename, document in documents.items() + } + ) ) - ) return files -def write_generated_files(*, check: bool) -> tuple[Path, ...]: +def write_generated_files( + *, check: bool, preserve_engine_lock: bool = False +) -> tuple[Path, ...]: """Write, or byte-check, the generated package resources.""" - expected = rendered_files() + expected = rendered_files(preserve_engine_lock=preserve_engine_lock) changed = tuple( path for path, payload in expected.items() @@ -445,13 +448,22 @@ def main(argv: list[str] | None = None) -> int: action="store_true", help="generate/check only engine_abi.lock.json; never rewrite bundle YAML", ) + parser.add_argument( + "--preserve-engine-lock", + action="store_true", + help="regenerate bundle domains while retaining the pending engine lock", + ) args = parser.parse_args(argv) + if args.engine_lock_only and args.preserve_engine_lock: + parser.error("--engine-lock-only and --preserve-engine-lock are incompatible") if args.engine_lock_only: changed = write_engine_abi_lock_only(check=args.check) if changed and not args.check: print("updated generated engine ABI lock") return 0 - changed = write_generated_files(check=args.check) + changed = write_generated_files( + check=args.check, preserve_engine_lock=args.preserve_engine_lock + ) if not args.skip_validation: digest = validate_generated_bundle() print(f"US bundle spec_sha256={digest}") diff --git a/tools/refresh_concept_coverage.py b/tools/refresh_concept_coverage.py index 29388e57bd..3bd97c241a 100644 --- a/tools/refresh_concept_coverage.py +++ b/tools/refresh_concept_coverage.py @@ -89,9 +89,38 @@ def _mapping_and_surface(engine: str) -> tuple[ConceptMapping, tuple[InputRef, . return axiom_concept_mapping(country), surface.refs() -def build_report(engine: str) -> CoverageReport: +def build_report( + engine: str, *, pending_us_e19200_leaves: bool = False +) -> CoverageReport: """The coverage report for ``engine`` from its mapping and surface.""" + if pending_us_e19200_leaves: + if engine != "policyengine-us": + raise ValueError("Pending E19200 leaves apply only to policyengine-us.") + from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, + ) + from microcosm.frame.adapters.policyengine_us_concepts import ( + POLICYENGINE_US_CONCEPT_MAPPING, + ) + + # The new engine release does not exist yet. Extend the committed + # reviewed surface through the normal coverage renderer, preserving + # its engine-version pin. A normal live-engine check still detects + # the pending names until the follow-up engine bump regenerates it. + pinned = CoverageReport.from_dict( + json.loads((GOLDEN / f"{engine}.json").read_text(encoding="utf-8")) + ) + surface = { + *pinned.covered_inputs, + *pinned.structural_inputs, + *pinned.uncovered_inputs, + *( + InputRef(name, "person") + for name in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS[:2] + ), + } + return coverage_report(POLICYENGINE_US_CONCEPT_MAPPING, surface) mapping, surface = _mapping_and_surface(engine) return coverage_report(mapping, surface) @@ -105,10 +134,17 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("--engine", choices=ENGINES, required=True, action="append") parser.add_argument("--check", action="store_true") + parser.add_argument( + "--pending-us-e19200-leaves", + action="store_true", + help="Extend the pinned US surface with points/premiums before the engine release; live-engine checks remain unchanged.", + ) args = parser.parse_args(argv) stale: list[str] = [] for engine in args.engine: - report = build_report(engine) + report = build_report( + engine, pending_us_e19200_leaves=args.pending_us_e19200_leaves + ) if report.unknown_inputs: labels = ", ".join(ref.label() for ref in report.unknown_inputs) print(f"{engine}: mapped inputs missing from the engine: {labels}") diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index c066d573ce..405cf37f19 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7" + "e91ed7916fe81a5dc1bff86a6f58511d6656eac977190f5a513f069af8a60701" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/contracts.py b/tools/us_bundle_generation/contracts.py index 9738516919..c8c73524d3 100644 --- a/tools/us_bundle_generation/contracts.py +++ b/tools/us_bundle_generation/contracts.py @@ -1236,14 +1236,23 @@ def resolve_calibration_tail_contracts( execution_tail = _mapping_like( binding.get("capital_gains_tail"), "primary PUF capital-gains tail" ) - soi = _mapping_like( - execution_tail.get("soi_e19200_agi_bands"), - "primary PUF SOI E19200 inputs", + soi = deepcopy( + dict( + _mapping_like( + execution_tail.get("soi_e19200_agi_bands"), + "primary PUF SOI E19200 inputs", + ) + ) ) + from microcosm.build.spec_engine.imputation_semantics import ( + restore_soi_e19200_agi_band_numbers, + ) + + restore_soi_e19200_agi_band_numbers(soi) runtime_agi_bands = dict( _mapping_like(soi.get("runtime_agi_bands"), "runtime SOI AGI bands") ) - runtime_sha256 = contract_sha256(runtime_agi_bands) + runtime_sha256 = runtime_agi_bands["sha256"] if soi.get("agi_bands") != runtime_agi_bands.get("agi_bands"): raise RuntimeError("SOI parsed and runtime AGI-band rows differ.") puf.update( diff --git a/tools/us_bundle_generation/core.py b/tools/us_bundle_generation/core.py index aa6448f100..0b22bc596c 100644 --- a/tools/us_bundle_generation/core.py +++ b/tools/us_bundle_generation/core.py @@ -36,6 +36,9 @@ from microcosm.build.spec_engine.publication_semantics import ( project_spine_legacy_sampling as _project_spine_legacy_sampling, ) +from tools.us_bundle_generation.pending_engine_inputs import ( + generator_variable_metadata, +) __all__ = [ "build_bundle", @@ -57,9 +60,7 @@ _CD_CROSSWALK_PACKAGE = "microcosm.build.us_runtime.data" _CD_CROSSWALK_RESOURCE = "congressional_district_vintage_crosswalk.csv" _CD_CROSSWALK_PROVENANCE_RESOURCE = f"{_CD_CROSSWALK_RESOURCE}.provenance.json" -_CD_CROSSWALK_SOURCE_ID = ( - "us_congressional_district_vintage_crosswalk_117_to_119" -) +_CD_CROSSWALK_SOURCE_ID = "us_congressional_district_vintage_crosswalk_117_to_119" _SHA256 = re.compile(r"^[0-9a-f]{64}$") _EXPECTED_INPUT_ROLES = frozenset( { @@ -191,6 +192,7 @@ def _publication_rung_rows( ) -> list[dict[str, int | float | str]]: return publication_rung_rows(publication) + def _compiled_publication_regex( *, pattern: str, @@ -203,11 +205,13 @@ def _compiled_publication_regex( rung_tokens=rung_tokens, ) + def project_publication_legacy_release( publication: Mapping[str, Any], ) -> dict[str, Any]: return _project_publication_legacy_release(publication) + def project_spine_legacy_sampling( spine: Mapping[str, Any], *, @@ -215,6 +219,7 @@ def project_spine_legacy_sampling( ) -> dict[str, Any]: return _project_spine_legacy_sampling(spine, publication=publication) + def _source_stage_compatibility() -> dict[str, Any]: from microcosm.build.source_manifest import SourceManifest @@ -254,9 +259,7 @@ def _congressional_district_vintage_crosswalk_provenance() -> dict[str, Any]: _CD_CROSSWALK_PROVENANCE_RESOURCE, ) expected_digest = payload.get("crosswalk_sha256") - if not isinstance(expected_digest, str) or not _SHA256.fullmatch( - expected_digest - ): + if not isinstance(expected_digest, str) or not _SHA256.fullmatch(expected_digest): raise ValueError("US CD-vintage crosswalk has no valid crosswalk_sha256.") raw = ( importlib_resources.files(_CD_CROSSWALK_PACKAGE) @@ -990,7 +993,8 @@ def build_vintages() -> dict[str, Any]: def build_catalogs() -> dict[str, Any]: """Build the complete physical column contract catalog. - PolicyEngine metadata covers the 142 modeled inputs. The producer graph + PolicyEngine metadata and reviewed pending declarations cover the modeled + inputs. The producer graph additionally emits generation-0 Frame structure, linkage, support provenance, and resolved weights which are deliberately outside the engine variable index; those contracts are derived from the canonical producer @@ -1013,7 +1017,7 @@ def build_catalogs() -> dict[str, Any]: columns: list[dict[str, Any]] = [] seen: set[str] = set() for entry in pool_input_surface(): - metadata = index.variable_metadata(entry.variable) + metadata = generator_variable_metadata(index, entry.variable) if metadata.entity != entry.entity: raise ValueError( f"Pool catalog entity differs for {entry.variable!r}: " @@ -1066,7 +1070,7 @@ def build_catalogs() -> dict[str, Any]: or output["column"] == "@resolved_weight" } missing_graph_contracts = graph_physical_keys - seen - if len(graph_physical_keys) != 134 or len(missing_graph_contracts) != 34: + if len(graph_physical_keys) != 136 or len(missing_graph_contracts) != 34: raise ValueError( "Canonical producer/catalog closure changed; " f"physical_outputs={len(graph_physical_keys)}, " @@ -1143,8 +1147,8 @@ def build_catalogs() -> dict[str, Any]: seen.add(key) if not deferred <= {key.split(".", 1)[1] for key in seen}: raise ValueError("Deferred pool inputs are absent from the generated catalog.") - if len(seen) != 176: - raise ValueError(f"Closed US catalog must contain 176 keys, got {len(seen)}.") + if len(seen) != 178: + raise ValueError(f"Closed US catalog must contain 178 keys, got {len(seen)}.") return { "metadata_waivers": [ { diff --git a/tools/us_bundle_generation/imputation.py b/tools/us_bundle_generation/imputation.py index 5e8a96bb37..ff43b8fda6 100644 --- a/tools/us_bundle_generation/imputation.py +++ b/tools/us_bundle_generation/imputation.py @@ -67,6 +67,9 @@ from microcosm.frame.adapters.policyengine_us import ( PolicyEngineUSVariableMetadataIndex, ) +from tools.us_bundle_generation.pending_engine_inputs import ( + generator_variable_metadata, +) _LEGACY_CLONE_ATTACHMENT_FRACTION = 1.0 _LEGACY_CLONE_ATTACHMENT_SEED = 578 @@ -413,7 +416,7 @@ def _target( entity: str, name: str, ) -> dict[str, object]: - declaration = metadata.variable_metadata(name) + declaration = generator_variable_metadata(metadata, name) if declaration.entity != entity: raise RuntimeError( f"Imputation target {entity}.{name} is owned by " @@ -2013,8 +2016,8 @@ def _split_after() -> list[dict[str, object]]: if group.entity == "person" and group.family.startswith("puf_tax_itemization__batch_") ] - if [len(group.targets) for group in late_batches] != [8, 8, 8, 8, 5]: - raise RuntimeError("Late PUF itemization is no longer the reviewed 5x37 split.") + if [len(group.targets) for group in late_batches] != [8, 8, 8, 8, 7]: + raise RuntimeError("Late PUF itemization is no longer the reviewed 5x39 split.") result = [ { "family": f"late/{group.entity}/{group.family}", @@ -2061,13 +2064,13 @@ def _assert_invariants( by_stage.setdefault(str(family["stage"]), []).append(family) expected = { "gap_fill_stacked_spine": (13, 48), - "primary_puf_qrf": (1, 65), - "late_producer_dag": (19, 70), + "primary_puf_qrf": (1, 67), + "late_producer_dag": (19, 72), } expected_dtypes = { "gap_fill_stacked_spine": {"bool": 20, "float": 27, "int": 1}, - "primary_puf_qrf": {"bool": 8, "float": 55, "int": 2}, - "late_producer_dag": {"bool": 17, "float": 50, "int": 1, "str": 2}, + "primary_puf_qrf": {"bool": 8, "float": 57, "int": 2}, + "late_producer_dag": {"bool": 17, "float": 52, "int": 1, "str": 2}, } for stage, (family_count, target_count) in expected.items(): rows = by_stage.get(stage, []) @@ -2093,7 +2096,7 @@ def _assert_invariants( len(compiled_schedule["edges"]), len(compiled_schedule["waves"]), len(graph["ownership_matrix"]), - ) != (38, 71, 6, 18): + ) != (38, 70, 5, 18): raise RuntimeError("US producer graph count invariant changed.") input_count = sum(len(node["inputs"]) for node in graph["nodes"]) authored_output_count = sum(len(node["outputs"]) for node in graph["nodes"]) @@ -2121,7 +2124,7 @@ def _assert_invariants( late_authored_output_count, len(tolerated_receipts), ) - expected_graph_counts = (2744, 92, 227, 35, 0, 213) + expected_graph_counts = (2748, 92, 231, 35, 0, 215) if graph_counts != expected_graph_counts: raise RuntimeError( "US producer graph input/output/absence counts changed: " diff --git a/tools/us_bundle_generation/pending_engine_inputs.py b/tools/us_bundle_generation/pending_engine_inputs.py new file mode 100644 index 0000000000..b40bf4aab6 --- /dev/null +++ b/tools/us_bundle_generation/pending_engine_inputs.py @@ -0,0 +1,39 @@ +"""Reviewed unreleased input declarations used only by bundle generation. + +The locked PolicyEngine-US version predates the two E19200 mortgage leaves. +Their declared Person/float/year contract lets this migration tool regenerate +the authored bundle before the follow-up engine repin. Runtime ownership and +consumer checks continue to use the installed engine without this fallback. +""" + +from microcosm.build.us_runtime.puf_interest_components import ( + US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS, +) +from microcosm.frame.adapters.policyengine_us import ( + PolicyEngineUSVariableMetadataIndex, +) +from microcosm.frame.schema import VariableMetadata + + +def generator_variable_metadata( + index: PolicyEngineUSVariableMetadataIndex, name: str +) -> VariableMetadata: + """Resolve installed metadata or the two reviewed pending declarations.""" + + if name in US_PUF_E19200_RESIDUAL_PERSON_OUTPUTS[:2]: + expected = VariableMetadata( + name=name, entity="person", dtype="float", period="year" + ) + try: + installed = index.variable_metadata(name) + except ValueError as error: + if str(error) != f"Unknown PolicyEngine-US source variable {name!r}.": + raise + return expected + if installed != expected: + raise ValueError( + f"Pending E19200 input {name!r} differs from its reviewed " + f"Person/float/year declaration: {installed!r}." + ) + return installed + return index.variable_metadata(name) From b29642a86c510b7f4d676ce320e6a74bd839ef96 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 7 Oct 2026 14:34:32 -0400 Subject: [PATCH 2/2] Refresh E19200 publication and authority receipt pins Keep strict publication and live authority comparisons current with the expanded E19200 QRF order and transfer target families. Pool fixture bytes and agreement remain unchanged. Co-Authored-By: GPT-6.1 Sol (subfleet lane) --- .../test_spec_engine_stacked_authority_semantics.py | 8 ++++---- .../engine_free/us/test_us_multispine_pool_tool.py | 12 +++++------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py index 284f3f613f..9ff94b12f7 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_stacked_authority_semantics.py @@ -76,7 +76,7 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer name: component["sha256"] for name, component in projected["components"].items() } == { "declared_surface": ( - "5b5a4470e2612365f253e933833bb08b8f9c857ea0bc175b958ead9a74abee01" + "a8a9a0e9a6f54dc36b99b5ce82fe2ebc224f72576e5c9ad4a73fd7cce60cbc6c" ), "gap_fill_plan": ( "f41319a95750a441676bc6599b1de6bb49a87b45d83b9263d627c402cfe8e750" @@ -85,13 +85,13 @@ def test_authority_projection_is_field_and_byte_identical_to_live_generation_zer "cacc6c11e114dbae3aaa2761cc6b3fcb1191cd9b689b1c2bd096614c51ebff8b" ), "late_producer_schedule": ( - "777979b267e7307581b39d14b7232b6ee2712e3da0029fffc090396ebd849166" + "4b8f521de3d19781ba76ebd0dbff4fa09c964e9978112e85ba5b8da32ef56af3" ), "metric_registry": ( - "d75cb9b29f8b0a9a085471a11f4c19c32ba04cbe5419053df94ea81cbe6125a9" + "fe7fe5ef7faa70ee0b06a59deda5542959dc0559a4ecaacd3dd0d79b28e5cb3d" ), "post_puf_transfer_surface": ( - "a31e8a9512ec829c98745ed9ca2177e66d529bdc4b096ecfa2b4452f7bd41d73" + "46a126ce2eb8303da9af3e2f3388a4a50794e2025ecaaade59e9cb3ea5edf469" ), "post_transfer_calibration": ( "141519684c72ab84a077ae0f5716a0416f1e19da57262948e459633cbe560576" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py index 19e6a40c39..cefa41ea84 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_multispine_pool_tool.py @@ -966,13 +966,11 @@ def deterministic_fixture_h5( # checkpoint metadata). "pool_h5": "ced797ecdd44a638c2a3945f07ad612098a7095ca53a5f458699bca6d6e38b3e", "agreement": "f39f0d918bf7ee01dddb5517d8830b8adb541273c5be084307be91397caca3cb", - # The engine-lock move to PolicyEngine-US 2.2.1 legitimately moves the - # pool-code checkpoint identities embedded in the otherwise legacy - # publication: the checkpoint identity carries policyengine_us_version, - # and the pool engine contracts it binds were re-derived for 2.2.1. - # pool_h5 and agreement above are unchanged, so only the identity - # surface moved, not the pool content. - "manifest": "e4692aa45f05826eb0097a7ae76dcbc712c13886a4d23a9c6a62b53752e323f1", + # The E19200 leaves change the pool-code checkpoint identity through + # the primary QRF order and transfer target families. + # Those identities also enter the legacy publication manifest; the + # fixture pool bytes and agreement above remain identical. + "manifest": "c0d547450e259c6dcf3e36bb24297161d87e6eed6b3f89240384cd66ab9f6409", }