diff --git a/changelog.d/us-esi-employer-premiums-454.added.md b/changelog.d/us-esi-employer-premiums-454.added.md new file mode 100644 index 0000000000..b6973abf21 --- /dev/null +++ b/changelog.d/us-esi-employer-premiums-454.added.md @@ -0,0 +1 @@ +The US base build produces `employer_sponsored_insurance_premiums` (microcosm #454). The `meps_esi_premiums` stage, declared but unable to run because the pinned ASEC inputs lack the Census policyholder fields, now reads `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP2`, `PEMLR` and `NOEMP` restored from the pinned Census person files. Every current ESI policyholder takes the MEPS-IC 2025 employer share for their coverage tier, firm size and State (census division for government employers), and one factor scales that whole universe to BEA NIPA Table 7.8 line 17 (B4923C, $977.0B for 2024), which counts contributions for active and retired employees (decision d1317). The column carries the employed policyholders' part ($863.9B on the default pool, before calibration); dependents, non-employed policyholders and the unincorporated self-employed carry zero. The US Chronicle feed is re-pinned to banked B4923C facts (Chronicle `cf9ece2`, three facts added, every other fact byte-identical), and the release tool refuses a feed whose values differ from the stage's. A release must carry the input with signal and its raw ASEC coverage columns, and hold the anchor-universe total within 5% of the anchor on the dense and the sparse export; the verdict ships as the `esi_premiums` gate evidence, with CMS NHE Table 24 and MEPS-IC's active-employee total as recorded cross-checks. The multispine pool does not run the stage yet, and an ACS local build refuses a frame whose input is null on its ACS rows. `pre_tax_health_insurance_premiums` is not produced: it waits for the engine's disjoint premium contract (policyengine-us #10046). diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 978c19dcd8..9b50ded052 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "44f40b4e8aaa2e6be67a7bc9680cc29f88e185c10573ffb9c3f9dfbd3ad1081f", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32437, "claim_count": 49, "claims": [ { @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1716, - "pointer_sha256": "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + "pointer_count": 1749, + "pointer_sha256": "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42217, + "consumed_field_count": 42217, "generation0_effect_counts": { - "legacy_behavior": 38498, + "legacy_behavior": 38531, "no_generation0_effect": 3686 }, "mode_counts": { "compiler_semantic": 27717, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14010 + "legacy_behavior": 14043 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", + "pointer_inventory_sha256": "c1e2a118a0d53db0bb628b14357a2c86db2956332f5b359c9092764f9c451a95", "resolved_binding_field_count": 9780, "unused_field_count": 0 }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" }, "status": "covered" }, @@ -1656,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", - "protocol_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68" + "map_sha256": "08698f4440eb31d1e2886036e7fd0ab86d4c6a007deaa650127dc4346a2151e0", + "protocol_sha256": "303da98866edb2a87b1dc36d9ff5a0146deb3a68a4f5361416b5794c13ae4cfb" }, "failures": [], "observed": { - "map_sha256": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", - "protocol_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68" + "map_sha256": "08698f4440eb31d1e2886036e7fd0ab86d4c6a007deaa650127dc4346a2151e0", + "protocol_sha256": "303da98866edb2a87b1dc36d9ff5a0146deb3a68a4f5361416b5794c13ae4cfb" }, "status": "covered" }, @@ -1677,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", + "implementation_sha256": "303da98866edb2a87b1dc36d9ff5a0146deb3a68a4f5361416b5794c13ae4cfb", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", + "implementation_sha256": "303da98866edb2a87b1dc36d9ff5a0146deb3a68a4f5361416b5794c13ae4cfb", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", "stages": 37 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "fb863f18a79610dc1005089a32e1535c502c5b9f7e5ba589db0d6025b01e3c82" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f" + "spec_sha256": "41d89d1b74bc3aec02b8dd9c3b3004928bfb0eface7bb670827289e75ab31ab8" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f" + "spec_sha256": "41d89d1b74bc3aec02b8dd9c3b3004928bfb0eface7bb670827289e75ab31ab8" }, "status": "pass" } diff --git a/docs/us-asec-census-person-columns.md b/docs/us-asec-census-person-columns.md index 9eef04b40a..11cd9aeefc 100644 --- a/docs/us-asec-census-person-columns.md +++ b/docs/us-asec-census-person-columns.md @@ -198,3 +198,43 @@ the Census recode. Census recode restored, `A_EXPRRP == 13` now appears in every vintage, but it still conflates partners and roommates, so the exclusion's conclusion stands. + +## Columns added for the ESI premium stage (#454) + +Added 2026-10-10. The `meps_esi_premiums` stage +(`docs/us-esi-employer-premiums.md`) reads six Census person columns that +**none** of the four pinned inputs carries, the 2024 and 2025 files included, +so the restoration appends them for every pooled vintage and never has an +input copy to verify against. + +| column | restored | reader | +|---|---|---| +| `NOW_OWNGRP` | yes | `esi_premiums._esi_person_codes` → the current employment-based policyholder | +| `NOW_HIPAID` | yes | same → employer paid all, some or none of the premium | +| `NOW_GRPFTYP2` | yes | same → coverage tier (family, self plus one, self-only) | +| `NOW_GRPFTYP` | yes | same → cross-checked against `NOW_GRPFTYP2` | +| `PEMLR` | yes | same → employed at interview | +| `NOEMP` | yes | same → employer size band | + +The reviewed domains are the Census API codebooks for survey years 2023 to +2026 (`api.census.gov/data//cps/asec/mar/variables/.json`), +which are identical across the four years for each column. The data also +carry `PEMLR == 0` (children and Armed Forces), which the codebooks omit, so +the `PEMLR` domain is 0 to 7. + +Observed in the pinned person members (rows, by exact `PERIDNUM` join): + +| income year | member | persons | policyholders (`NOW_OWNGRP` 1) | `NOW_HIPAID` all / some / none | `NOW_GRPFTYP2` family / self plus one / self-only | employed (`PEMLR` 1-2) | +|---|---|---:|---:|---|---|---:| +| 2022 | `pppub23.csv` | 146,133 | 39,838 | 8,470 / 27,889 / 3,479 | 10,677 / 7,652 / 21,509 | 68,695 | +| 2023 | `pppub24.csv` | 144,265 | 39,161 | 7,843 / 28,047 / 3,271 | 10,393 / 7,386 / 21,382 | 68,511 | +| 2024 | `pppub25.csv` | 142,125 | 38,554 | 7,748 / 27,378 / 3,428 | 10,176 / 7,228 / 21,150 | 67,436 | +| 2025 | `pppub26.csv` | 134,729 | 36,185 | 7,449 / 25,717 / 3,019 | 9,623 / 6,806 / 19,756 | 63,429 | + +In every member the payment-status and tier counts sum to the policyholder +count, and everyone else carries 0 in both. The stage refuses a frame where +that does not hold. + +The older note that the 2024 and 2025 inputs "carry all 18" `NOW_*` recodes +counts the coverage-type recodes reviewed for #720. `NOW_OWNGRP`, +`NOW_HIPAID`, `NOW_GRPFTYP` and `NOW_GRPFTYP2` were never among them. diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index 26702e36c3..713d43b640 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -13,6 +13,53 @@ any other feed, and `test_us_chronicle_feed.py` fails if the parity resources, the generator and the pin disagree, or if the scope file changes without a new pin. +## The 2026-10-11 re-pin: BEA private group health insurance (microcosm#454) + +Decision d1317 made BEA NIPA Table 7.8 line 17 (series B4923C) the scale +anchor of the `meps_esi_premiums` stage (`docs/us-esi-employer-premiums.md`) +and asked for the feed to carry the banked fact. The pin moved from `51aa40f` +to `cf9ece2`, a commit that is `51aa40f` plus two commits (Chronicle branch +`feed/us-51aa40f-esi-b4923c`, tag `microcosm-us-feed-454-v1`), not Chronicle +main: + +- The digit-text guard match from Chronicle main's `aa3e4de3`, and nothing + else from that commit. BEA's Section 7 workbook writes its year headers as + text (`'2024'`), and at `51aa40f` no guard could match them. Only a guard + that used to fail can pass, so every package that built before builds the + same cells. +- The `bea-nipa-private-group-health-insurance` package as + PolicyEngine/chronicle#331 merged it to main, with its concept evidence + (BEA's estimates cover active and retired employees). The test file drops + main's artifact-year restamp test, whose guard (chronicle#292) postdates + this base, and the package is not added to the US poverty hard-target + coverage contract, as on main. + +The scope adds three pairs, `bea_nipa.cy.private_group_health_insurance` +for 2023, 2024 and 2025, each built at its own year (640 pairs, 66 package +runs). Built at `cf9ece2`: + +- the feed keeps all 40,178 `51aa40f` rows byte for byte (joined on + `aggregate_fact_key`) and adds exactly the three B4923C facts: $923,195M, + $977,034M and $1,027,929M, each from `Section7All_xls.xlsx` (sha256 + `de1c34e3…`); +- compiled at 2024 with the packaged CD crosswalk and the Medicaid + substitutions, with target aging on and again with it off, the registry is + the same ordered list of targets on both feeds, name, value and metadata + (33,250 targets with aging on). B4923C is not compiled into a target: the stage scales to it, + and whether to calibrate the column is decision d1264. The target-parity + manifest lists it as a `deferred` reviewed exclusion (82 feed families, 32 + compiled, 53 reviewed exclusions; it was a source-absent family before); +- two independent rebuilds gave the same `facts_sha256`. + +The comparison scripts and their reports are in +`experiments/us-esi-454/feed-repin/`. The release tool now refuses a feed +whose B4923C values differ from the stage's pinned anchor +(`us_esi_premiums_anchor_feed_mismatches`); a diagnostic run under +`--allow-unpinned-feed` or `--allow-esi-premium-gaps` continues and records +the difference in `esi_premiums.json` (in the report, for a dry run). A later re-pin that changes other rows +(microcosm#1192 re-dates the W-2 item facts) must carry the three B4923C +pairs and redo the row comparison against its own previous feed. + ## The 2026-10-04 re-pin: TY2023 state AGI bands (microcosm#940) The state x AGI-band targets of #940 need the TY2023 Historic Table 2 state @@ -54,20 +101,22 @@ the first labelled US export. | Field | Value | |---|---| -| Chronicle commit | `51aa40fd28e2ca81937752515e773d123fecfb71` (`c5e5bf8` plus the TY2023 state AGI package; tag `microcosm-us-feed-940-v1`) | -| Feed file | `consumer_facts_us_51aa40f.jsonl`, 40,178 rows, 169,038,757 bytes | -| `facts_sha256` | `965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac` | +| Chronicle commit | `cf9ece2e7be11158c21fdc4dd38b44d92db552f6` (`51aa40f` plus the digit-text guard and the B4923C package; tag `microcosm-us-feed-454-v1`) | +| Feed file | `consumer_facts_us_cf9ece2.jsonl`, 40,181 rows, 169,055,585 bytes | +| `facts_sha256` | `054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57` | | Consumer fact schema | `chronicle.consumer_fact.v3`, schema file sha256 `bdb51e2a…` (unchanged from the UK pin) | -| Scope | 637 (record set, period) pairs; 63 package runs over build years 2020 to 2029 | +| Scope | 640 (record set, period) pairs; 66 package runs over build years 2020 to 2029 | | Consumer artifact | none: refused at this commit, see below | -The previous pin was `c5e5bf8aa84960c1a200ee47303b19c953092d0f` -(`consumer_facts_us_c5e5bf8.jsonl`, 39,158 rows, `facts_sha256` -`b8543739…`, 586 pairs, 62 runs); the rest of this document describes its -2026-09-18 rebuild unless it says otherwise. +The previous pins were `51aa40fd28e2ca81937752515e773d123fecfb71` +(`consumer_facts_us_51aa40f.jsonl`, 40,178 rows, `facts_sha256` +`965a29ac…`, 637 pairs, 63 runs) and, before it, +`c5e5bf8aa84960c1a200ee47303b19c953092d0f` (`consumer_facts_us_c5e5bf8.jsonl`, +39,158 rows, `facts_sha256` `b8543739…`, 586 pairs, 62 runs); the rest of this +document describes the 2026-09-18 rebuild unless it says otherwise. The feed is too large for the repository. Its home on the build machine is -`~/PolicyEngine/_buildh-runtime/inputs/consumer_facts_us_51aa40f.jsonl`, +`~/PolicyEngine/_buildh-runtime/inputs/consumer_facts_us_cf9ece2.jsonl`, beside the previous pins; `tools/build_us_target_parity_manifest.py` reads it there by default. A holder of the Chronicle commit regenerates it byte for byte with the commands below. diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md new file mode 100644 index 0000000000..d6b3f92c40 --- /dev/null +++ b/docs/us-esi-employer-premiums.md @@ -0,0 +1,381 @@ +# US employer-sponsored insurance premiums (#454) + +Status as of 2026-10-11. Code: `microcosm.build.us_runtime.esi_premiums` +(stage `meps_esi_premiums`). Receipts: `experiments/us-esi-454/`. + +## What the stage produces + +One PolicyEngine-US person input that no Microcosm stage produced, +`employer_sponsored_insurance_premiums`. Its consumer was read in the locked +engine, policyengine-us 2.2.1: `gov.household.cbo_market_income_additions`, +summed into `cbo_household_market_income`. No other variable or parameter +list names it. + +`tests/engine_contract/us/test_us_esi_premiums_engine.py` computes it in the +engine for one worker: a $10,000 employer premium raises +`cbo_household_market_income` by exactly $10,000 and moves none of the tax +measures. + +It was a recorded parity gap: the retired eCPS derivation (archived +`42ed5d45`, `datasets/cps/cps.py` L197-271) read the CPS ASEC current +employment-based coverage fields, and every pinned processed ASEC input omits +them. + +The stage does not produce `pre_tax_health_insurance_premiums`, the employee +share paid by pre-tax payroll deduction. See "The pre-tax employee premium" +below. + +## Source fields + +`asec_census_person_columns` now restores six more Census person columns into +every pooled vintage, by exact one-to-one `PERIDNUM` identity from the pinned +official person members (`docs/us-asec-census-person-columns.md`). None of the +four pinned inputs (income years 2022 to 2025) carries any of them. + +| Column | Use | Codes | +|---|---|---| +| `NOW_OWNGRP` | current employment-based policyholder | 1 yes, 2 no, 0 out of universe | +| `NOW_HIPAID` | employer paid the premium | 1 all, 2 some, 3 none, 0 out of universe | +| `NOW_GRPFTYP2` | coverage tier | 1 family, 2 self plus one, 3 self-only | +| `NOW_GRPFTYP` | cross-check on the tier | 1 family, 2 self-only | +| `PEMLR` | employed at interview | 1 at work, 2 absent | +| `NOEMP` | employer size (the work-experience item "persons who work for employer") | 1 under 10, 2 10-49, 3 50-99, 4 100-499, 5 500-999, 6 1000+, 0 not in universe | + +The codes are the Census API codebooks for survey years 2023 to 2026, which +are identical across the four years for all six columns, and were checked +against the four pinned person members (`pppub23` to `pppub26`). In each file: + +- a policyholder always reports a payment status and a tier, and nobody else + reports either; +- `NOW_GRPFTYP` family equals `NOW_GRPFTYP2` family or self plus one, row for + row. + +The stage refuses any frame that breaks these two relations, carries a missing +or out-of-codebook value, or lacks a column. It never defaults a value. It +does not read `NOW_GRP` or `has_esi`: neither identifies the policyholder. (In +the four files every policyholder also has `NOW_GRP == 1`; that was observed, +and the stage does not enforce it.) + +`PEIO1COW` (class of worker) is in the pinned inputs already. State is the +household `GESTFIPS`. + +## Employer premium + +### Two universes + +The national accounts that publish an employer-contribution total count +contributions for active **and** retired employees (see the anchor section). +The stage therefore keeps two universes apart: + +- **Anchor universe: every current policyholder** (`NOW_OWNGRP` 1), employed + or not. Each is priced from MEPS-IC, and one factor scales this universe to + the anchor. +- **Column universe: employed policyholders with an employer** (`PEMLR` 1 or + 2, `PEIO1COW` 1 to 6). Only they carry + `employer_sponsored_insurance_premiums`, at the same factor. + +Everyone else carries zero in the column: + +- dependents, because the policyholder carries the premium; +- retired, unemployed and other non-employed policyholders, because their + coverage is not compensation of a current job; +- the unincorporated self-employed and workers without pay (`PEIO1COW` 7 and + 8), whose current job is not an employer's job. Their group coverage is + priced in the anchor universe, like a retiree's, and left out of the + column. + +The non-employed policyholders' scaled share is priced and then left out. It is +not loaded onto workers. + +### Cells + +MEPS-IC average total premium and average employee contribution per enrolled +employee, by coverage tier: + +- Private employers (`PEIO1COW` 4 to 6): MEPS-IC 2025 Series II, by State and + firm size. `NOEMP` 1 and 2 take the under-50 column, 3 to 6 the 50-or-more + column, and 0 (not in universe) the State's all-sizes column. MEPS-IC's + finer bands (10-24, 25-99, 100-999) do not nest inside the `NOEMP` bands, so + they cannot be used. +- State government (`PEIO1COW` 2): MEPS-IC 2024 Series III, State-government + column, by census division. +- Local and federal government (`PEIO1COW` 3 and 1): the same table's + all-governments column. MEPS-IC does not survey the federal government; this + is a stand-in. +- No employer class (`PEIO1COW` 0, 7 or 8): the State's all-sizes private + column. A non-employed policyholder who does report a class of worker keeps + that employer's cell. + +`NOEMP` and `PEIO1COW` describe different jobs. `PEIO1COW` is the class of +worker of the current job. `NOEMP` is the size of the employer of the longest +job held last year. A worker whose longest job last year was with a different +employer from the current one takes that other employer's size band, and a +current worker with no job last year (`NOEMP` 0) takes the State's all-sizes +column. + +Series III for 2025 is not yet published. The 2024 government cells are aged +to 2025 by the private-sector national ratio of the same tier and measure +(for example single premium, 9,025 / 8,486). + +AHRQ suppresses nine small-firm employee-contribution cells in the 2025 State +tables (family: AK, DE, NM, SC, WV; self plus one: AK, KY, SC, SD). Each takes +the national small-firm cell times the State's all-sizes level relative to the +nation. No premium cell the stage reads is suppressed. + +No MEPS-IC table prices retiree coverage, so a non-employed policyholder is +priced as an active employee. If retiree plans cost less than active plans, +the employed column is understated; the receipts carry the sensitivity. + +### Share by payment status + +- Employer paid all (`NOW_HIPAID` 1): the cell premium. +- Employer paid some (2): the premium minus the contribution of an employee + who pays one. +- Employer paid none (3): zero. + +MEPS-IC's average employee contribution is taken over every enrollee, +including those whose coverage required no contribution. Subtracting it from +"paid some" holders only, while "paid all" holders keep the whole premium, +would overstate the cell's employer mean. The stage divides the average +contribution by one minus the published share of enrollees with no +contribution (MEPS-IC 2025 Tables II.C.4.a, II.D.4.a and II.E.4.a, United +States row by firm size): + +| Tier | All sizes | Under 50 | 50 or more | +|---|---:|---:|---:| +| Self-only | 13.0% | 35.5% | 8.7% | +| Family | 7.7% | 28.6% | 4.6% | +| Self plus one | 6.0% | 23.1% | 3.6% | + +A cohort with MEPS-IC's own mix of the two groups then reproduces the cell's +published employer mean (premium minus average contribution) exactly. For the +national under-50 self-only cell, premium $9,034 and average contribution +$1,924, the unconditional rule gives $7,793 and the conditional rule the +published $7,110. + +Limits of this rule: + +- The no-contribution shares are national. Most State cells of those tables + are flagged unreliable or suppressed. +- Series III publishes no such share, so government cells take the private + 50-or-more share. +- CPS respondents say "employer paid all" more often than MEPS-IC + establishments report a zero contribution (18.7% of employed policyholders, + against the shares above). On the pinned pools the stage's raw mean among + employer-paying workers is therefore 3.1% above the published cell means + (`raw_over_published_employer_mean`), before the national factor. +- MEPS-IC publishes no share of enrollees who pay the whole premium. The 7.2% + of employed policyholders who say their employer pays nothing carry zero and + are treated as outside the MEPS-IC averages. + +### Scale + +One factor scales every policyholder's share so that the weighted +anchor-universe total equals the anchor for the build year. The column is that +factor times the employed policyholders' shares. + +## The anchor + +| Series | CY2024 | Who it covers | +|---|---:|---| +| BEA NIPA Table 7.8 line 17, B4923C, "Private group health insurance" | **$977.0B** | Employer contributions as a supplement to wages and salaries. BEA's State Personal Income methodology (December 2025, paragraphs 3.18 and 3.19) says its MEPS source "covers both health insurance purchased by employers for their active and retired employees" and that the federal estimate covers "active and retired federal civilian employees". | +| CMS NHE Table 24, "Employer Contribution to Private Health Insurance Premiums" | $1,047.0B | Private, federal and State and local employers. The NHEA methodology paper (2024, p. 29) says the estimates cover "active employees, continuation of health coverage (COBRA), and retirees" of private and State and local sponsors, from MEPS-IC, and federal "employers, employees and retirees", from OPM. | + +Both series measure the same concept, and both include retirees. The stage +scales to **BEA**, as decision d1317 (2026-10-11) ruled. **NHE** is recorded +beside every anchor verdict and is not gated: it is 7.2% above BEA for 2024, +more than the anchor gate's tolerance, so a release scaled to NHE would fail +that gate. + +The stage pins BEA's values itself (`EMPLOYER_PREMIUM_ANCHOR`, the +Section 7 workbook by URL and SHA-256). The pinned US Chronicle feed banks the +same series (record sets `bea_nipa.cy.private_group_health_insurance`, +PolicyEngine/chronicle#331; `docs/us-chronicle-feed-repin.md`). Two checks +hold them together. A test compares the anchor with the three facts the +committed re-pin report says the pinned feed added, so changing one without +the other fails CI. And the release tool refuses a feed whose values differ +from the stage's (`us_esi_premiums_anchor_feed_mismatches`); a diagnostic run +under `--allow-unpinned-feed` or `--allow-esi-premium-gaps` continues and +records the difference in `esi_premiums.json`, or in the report of a +`--dry-run-gates-report` run, which stops before that file is written. + +Two facts differ from what #454 recorded in July: + +1. BEA revised 2024. The issue cited $1,002.9B, from the release of + 2025-09-26. The annual update published 2026-09-30 puts 2024 at $977.034B + (2023 $923.195B, unchanged; 2025 $1,027.929B). +2. The issue described NHE as the broader series because it includes + retirees. BEA's does too. + +The workbook column covers 2023 to 2025, so the stage refuses any other build +year. + +### What the choice moves + +The receipts restate the default pool's column under each defensible reading. +The anchor and the treatment of non-employed policyholders are methodology +choices; the first row is what the stage builds. + +| Reading | Employed column | Mean per policyholder with a positive premium | +|---|---:|---:| +| **BEA over every policyholder (as built, d1317)** | **$863.9B** | **$11,679** | +| BEA over every policyholder, 65-and-over non-employed at half price | $884.7B | $11,961 | +| NHE over every policyholder | $925.7B | $12,515 | +| BEA loaded onto employed policyholders only | $977.0B | $13,209 | +| NHE loaded onto employed policyholders only | $1,047.0B | $14,155 | + +One outside check bears on it, and decided it. MEPS-IC's own national rows +give the private-sector employer total for enrolled, active employees: +employees times the share in establishments that offer insurance times the +share enrolled there, times each tier's share of enrollees and its premium +less average contribution. That is $668.6B for 2024 (65.6M enrollees) and +$714.4B for 2025. The private-sector part of the as-built column is $665.6B, +0.4% below the 2024 figure; under the NHE reading it would be $713.2B, 6.7% +above. The check covers the private sector only (77% of the column) and +shares the cells' source. + +## The pre-tax employee premium + +`pre_tax_health_insurance_premiums` has no producer, and this stage does not +add one. + +PolicyEngine-US treats the premium inputs as disjoint: a premium paid by +pre-tax payroll deduction belongs only in `pre_tax_health_insurance_premiums`, +and `health_insurance_premiums_without_medicare_part_b`, +`other_health_insurance_premiums` and `health_insurance_premiums` hold the +premiums not paid that way (PolicyEngine/policyengine-us#10046). The base +carries the whole reported premium (`PHIP_VAL`) in the second set. A producer +therefore has to move the pre-tax dollars out of the reported premium, not +copy them: + +- Copying them would count the same premium twice. The engine would exclude + it from wages and still deduct it for itemizers with medical expenses above + the floor (for a filer with $60,000 of wages, $20,000 of other medical + expenses and a $2,000 premium, income tax of $4,445.74 against $4,685.74). +- Moving them is only right on an engine that counts the pre-tax input where + the reported premium was counted. policyengine-us 2.2.1 does not: its + `spm_unit_health_insurance_premiums` adds `other_health_insurance_premiums` + and the modeled premiums, not the pre-tax input, so the moved dollars would + drop out of SPM medical out-of-pocket expenses. #10046 adds it. +- Moving about $180 billion out of two premium inputs also changes what the + release tool's input-mass parity gate and the eCPS parity reference expect + of those columns. + +An earlier round of this work assigned the reported premium, capped at wages, +to eligible workers at MEPS-IC pretax-offer rates and measured $180.1 billion +over 52.5 million workers on the default pool. That code is on branch +`esi-pre-tax-premiums-454` and is the starting point for the follow-up, which +is sequenced after an engine pin that includes #10046. + +## Measurements on the pinned inputs + +`experiments/us-esi-454/run_stage_on_pinned_pool.py` pools the real pinned +inputs the way the base build does and runs the stage. These are +pre-calibration figures at pooled ASEC weights, target year 2024. + +| | 2023-2025 (default) | 2022-2024 | +|---|---:|---:| +| Current ESI policyholders | 92.4M | 92.9M | +| Employed, with an employer | 79.7M | 79.8M | +| With a positive employer premium | 74.0M | 74.1M | +| Other policyholders (priced, outside the column) | 12.7M | 13.2M | +| Raw MEPS-IC total, every policyholder | $1,006.4B | $1,012.1B | +| Scale factor | 0.971 | 0.965 | +| Anchor-universe total | $977.0B | $977.0B | +| **Employer premium column** | **$863.9B** | $862.2B | +| Employed share of the anchor universe | 88.4% | 88.2% | +| Other policyholders' share, not in the column | $113.2B | $114.9B | +| Mean per policyholder with a positive premium | **$11,679** | $11,642 | +| Mean per employed policyholder | $10,840 | $10,805 | +| By tier: family / self plus one / self-only | $366.9B / $184.9B / $312.1B | $366.9B / $186.3B / $308.9B | +| By employer: private / State and local / federal | $665.6B / $162.7B / $35.6B | $663.7B / $162.5B / $36.0B | + +Of the 12.7M other policyholders in the default pool, 7.1M are retired +(`PEMLR` 5), 2.1M are otherwise out of the labor force (7), 0.9M are disabled +(6), 1.0M are unemployed (3 and 4), 1.5M are employed without an employer +class, and 0.1M have no labor-force status (0). + +Both the signal gate and the anchor gate pass on each pool, before and after +the PUF-support clone, and the clone conserves both totals. + +## Gates + +| Gate | Where | Checks | +|---|---|---| +| `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Column present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible; no employer premium outside the column universe or where the employer pays none; every premium is one common multiple of its MEPS-IC share. | +| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of BEA NIPA 7.8 line 17 for the release period. Employed policyholders must carry 80% to 95% of it. | +| `us_release_input_coverage_gate` | release tool | The input is `required` with no reviewed exclusion, so an export that drops or flattens it fails. | +| `assert_required_us_release_source_columns` | the L0/refit export (`l0_refit_export.export_us_l0_refit_h5`, run by `tools/export_us_l0_refit_h5.py`), unless `--allow-missing-source-columns` is passed | The column must be present and non-constant. | +| reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B. | + +What the gates need, and what they do not catch: + +- **They need the raw ASEC columns.** Both gates recompute the assignment + from `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP`, `NOW_GRPFTYP2`, `PEMLR`, + `NOEMP`, `PEIO1COW` and State. A frame missing any of them fails both: it + cannot be certified. Published release files carry raw ASEC person fields + today (`PEIO1COW`, `PHIP_VAL`, `WSAL_VAL`, `NOW_MCAID`); a release path + that dropped the restored columns would turn both gates red. +- **The raw-mean band catches unit errors only.** The signal gate requires + the unscaled share per holder to sit between $6k and $20k, which catches a + cell table in the wrong units. A cell table that is 20% low passes: the + scale factor absorbs it. A re-pinned table is checked against the PDFs by + `tools/build_us_meps_ic_esi_cells.py --check`, not by this gate. +- **The 5% tolerance is a chosen criterion.** No release has calibrated this + column yet, so the tolerance is not derived from measured drift. The first + release build records the drift in `esi_premiums.json`. + +The stage pins the pre-calibration total exactly. Nothing calibrates the +column, so the anchor gate is what holds a release's total after reweighting +and sparse selection. `esi_premiums.json` is required release evidence: it +carries the anchor verdict, both cross-checks, the base and export signal +verdicts and the anchor-versus-feed verdict, and both manifests record it. +`--allow-esi-premium-gaps` stops the gates failing a diagnostic build, for +example on a base built before this stage; the failures it waives, including +a feed whose B4923C facts differ from the anchor, are still written to that +file. + +## Lineage + +- Census person members: pinned in + `education_assistance_source.ASEC_EDUCATION_ASSISTANCE_ARCHIVES` (archive + and member SHA-256), read by the #720 restoration. +- MEPS-IC cells: three AHRQ PDFs pinned by URL, byte length and SHA-256 in + `tools/build_us_meps_ic_esi_cells.py`, extracted to + `us_runtime/data/meps_ic_esi_premium_cells.json`, whose own SHA-256 the + stage pins and checks on load. +- BEA and NHE files: URL, byte length and SHA-256 in the stage manifest + (`us/source_stages.json`), and URL and SHA-256 in the module; the BEA + values also as Chronicle feed facts, checked against the module by the + release tool. The NHE + digest is of the zip archive, not of the Table 24 workbook inside it. +- The base summary records the stage's full summary twice + (`esi_premiums_signal`, `esi_premiums_post_clone_signal`): scale factor, raw + and scaled totals for both universes, totals by tier and employer sector, + weighted policyholder counts, the anchor, the cross-check and the cell-table + digest. + +## What this does not do + +- **The multispine pool.** `multispine_pool.py` runs its own operator chain on + CPS-source rows that carry only part of the stacked population's weight, so + scaling to a national total there needs its own design. Until then the + coverage and anchor gates are red on that lineage. The raw-stage operator + boundary already refuses a source frame that carries the output. +- **ACS local releases.** An ACS spine pooled beside a donor that carries the + input has it null on its ACS rows. `fill_reviewed_nulls` refuses that frame + instead of default-filling it to zero, so this lineage does not build until + the input is assigned or transferred for every spine. +- **A calibration target.** The column is the employed part of the BEA fact, + so the fact cannot be compiled as a sum target on it. The target-parity + manifest keeps the BEA family and the two `cms_nhe` ESI families as + `deferred` reviewed exclusions. Whether and how to calibrate the column is + decision d1264. +- **The standalone L0/refit export.** `tools/export_us_l0_refit_h5.py` + writes an H5 without running the signal or anchor gate. By default it + refuses a frame whose column is absent or constant, and its generic + input-mass parity check (default tolerance 50%) compares the column's + weighted total with the base or reference, as for every input. Neither ESI + gate runs there: they run only in the base build and in + `tools/build_us_fiscal_refresh_release.py`. +- **The pre-tax employee premium.** See the section above. diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index f3f62e66aa..507b2dd70c 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -23,7 +23,7 @@ sequence. | Boundary | Operators | State consumed | |---|---|---| | `source_construction` | `_load_base_frame_from_args` | Either an existing US base H5 or pooled ASEC unit frames built from the declared source years. | -| `pre_clone_enrichment` | `derive_us_cps_carried_inputs`; prior-year income, relationships, Medicare take-up, housing, eligibility, pregnancy, WIC, child support, disability benefits, workers compensation, weeks unemployed, childcare, energy subsidy, retirement contribution/distribution, and immigration operators | The ASEC-only frame, build year and seed; some operators also consume pinned external source tables. The adjacent-year income join and rent draw happen here, before row expansion. Housing can consume the pinned ACS 2022 rent donor when its input gate is not already satisfied. | +| `pre_clone_enrichment` | `derive_us_cps_carried_inputs`; prior-year income, relationships, Medicare take-up, housing, eligibility, pregnancy, WIC, child support, disability benefits, workers compensation, weeks unemployed, childcare, energy subsidy, retirement contribution/distribution, ESI premiums (#454), and immigration operators | The ASEC-only frame, build year and seed; some operators also consume pinned external source tables. The adjacent-year income join and rent draw happen here, before row expansion. Housing can consume the pinned ACS 2022 rent donor when its input gate is not already satisfied. | | `clone_feature_extraction` | `clone_us_frame_for_puf_support`; PUF donor extraction; primary-QRF initialization | The already enriched ASEC frame and processed PUF donor arrays. Cloning creates an ASEC copy and a PUF-tax-detail copy and splits weights across them. | | `primary_qrf_chain` and `qrf_finalization` | Chained weighted QRF fits and finalization | Predictors on the cloned frame, PUF donor targets, design weights, fit seed, and estimator count. Predictions are assigned to the PUF-detail copy. | | PUF tail and derived-detail stages | Capital-gains tail transfer, capital-gain distributions, QBI reconciliation | The cloned, QRF-imputed frame plus PUF donor detail and declared deterministic reconciliation rules. | diff --git a/experiments/us-esi-454/README.md b/experiments/us-esi-454/README.md new file mode 100644 index 0000000000..1947636e30 --- /dev/null +++ b/experiments/us-esi-454/README.md @@ -0,0 +1,52 @@ +# ESI premium stage on the pinned ASEC pools (#454) + +`run_stage_on_pinned_pool.py` pools real pinned ASEC inputs the way the base +build's source construction does (`build_pooled_asec_unit_frame`, with the +Census person-column restoration), runs the `meps_esi_premiums` stage, its +signal gate and the release anchor gate, clones the frame for PUF support and +grades the clone, and writes what it measured to `receipts/`. + +``` +uv run python tools/fetch_us_asec_sources.py --all-pinned # prints the pinned paths +uv run python experiments/us-esi-454/run_stage_on_pinned_pool.py \ + --asec-h5 2023= --asec-h5 2024= \ + --asec-h5 2025= \ + --census-person-dir ~/.cache/microcosm/cps/asec_education +``` + +The script refuses an input whose SHA-256 is not its pin +(`asec_sources.ASEC_SOURCE_ARTIFACTS`, `spm_role_source.ASEC_SPM_ROLE_SOURCES`). +About 80 seconds per pool. + +| Receipt | Pool | +|---|---| +| `receipts/stage_on_pool_2023_2025.json` | the default pool (`docs/us-asec-source-pins.md`) | +| `receipts/stage_on_pool_2022_2024.json` | the historical Build J/N/P pool | + +Each receipt records the input digests, the columns the restoration added per +vintage with their observed code counts, the stage summary (scale factor, raw +and scaled totals for both universes, totals by tier and employer sector, +weighted policyholder counts), totals by vintage, both gate verdicts before +and after the clone with the BEA and MEPS-IC cross-checks, and +`other_policyholder_sensitivity`: the employed column under each reading of +the anchor and of how policyholders outside the column are priced. + +These are pre-calibration figures at pooled ASEC weights. They are evidence +about the stage, not about a release: +`packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py` pins +the figures the docs and registers quote to these files. + +## The feed re-pin + +`feed-repin/` holds the two checks behind the 2026-10-11 re-pin of the US +Chronicle feed to `cf9ece2`, which banks BEA B4923C, the stage's anchor +(`docs/us-chronicle-feed-repin.md`): + +- `compare_feeds.py` joins the previous and new feeds on + `aggregate_fact_key` and compares row bytes; `value_diff.json` records 0 + rows missing, 0 changed and the three added B4923C facts. +- `compare_registries.py` compiles the US fiscal target registry on both + feeds, once the way the target-parity generator does (targets aged) and + once with aging off, and compares each as an ordered list; + `registry_diff.json` records 33,250 targets with distinct names on each + feed and identical lists under both settings. diff --git a/experiments/us-esi-454/feed-repin/compare_feeds.py b/experiments/us-esi-454/feed-repin/compare_feeds.py new file mode 100644 index 0000000000..5c8facf7b7 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/compare_feeds.py @@ -0,0 +1,72 @@ +"""Compare two US Chronicle feeds row by row (microcosm#454 re-pin). + + uv run python experiments/us-esi-454/feed-repin/compare_feeds.py \ + + +Joins on ``aggregate_fact_key`` and compares each row's exact bytes. The +re-pin is accepted only if no previous row is missing or changed; the report +lists every added row's record set, period, value and source digest. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + + +def _rows(path: Path) -> dict[str, bytes]: + rows: dict[str, bytes] = {} + with path.open("rb") as stream: + for line in stream: + key = json.loads(line)["aggregate_fact_key"] + if key in rows: + raise SystemExit(f"{path}: repeated aggregate_fact_key {key}") + rows[key] = line + return rows + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def main() -> None: + previous_path, new_path, report_path = (Path(arg) for arg in sys.argv[1:4]) + previous, new = _rows(previous_path), _rows(new_path) + missing = sorted(key for key in previous if key not in new) + changed = sorted( + key for key in previous if key in new and previous[key] != new[key] + ) + added = [] + for key in sorted(set(new) - set(previous)): + row = json.loads(new[key]) + added.append( + { + "aggregate_fact_key": key, + "record_set_id": row["layout"]["record_set_id"], + "period": row["period"], + "value": row["value"], + "source_sha256": row["source"].get("source_sha256"), + } + ) + report = { + "previous": { + "file": previous_path.name, + "sha256": _sha256(previous_path), + "rows": len(previous), + }, + "new": {"file": new_path.name, "sha256": _sha256(new_path), "rows": len(new)}, + "previous_rows_missing": len(missing), + "previous_rows_changed": len(changed), + "rows_added": added, + } + Path(report_path).write_text(json.dumps(report, indent=1, sort_keys=True) + "\n") + print(json.dumps({k: v for k, v in report.items() if k != "rows_added"}, indent=1)) + if missing or changed: + raise SystemExit("previous rows missing or changed") + + +if __name__ == "__main__": + main() diff --git a/experiments/us-esi-454/feed-repin/compare_registries.py b/experiments/us-esi-454/feed-repin/compare_registries.py new file mode 100644 index 0000000000..261cac26d0 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/compare_registries.py @@ -0,0 +1,93 @@ +"""Compile the US fiscal target registry on two feeds and compare (microcosm#454). + + uv run python experiments/us-esi-454/feed-repin/compare_registries.py \ + + +Compiles at target period 2024 with the canonical congressional-district +crosswalk and the reviewed Medicaid substitutions, once the way +tools/build_us_target_parity_manifest.py does (targets aged) and once with +aging off (dollar targets allowed unaged). Each registry is compared as an +ordered list, so a repeated, reordered, added, dropped or changed target shows +up; every field of every target is serialized. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + +CONFIGURATIONS = { + "aged": {"age_targets": True}, + "unaged": {"age_targets": False, "allow_unaged_dollar_targets": True}, +} + + +def _generator(): + path = ROOT / "tools" / "build_us_target_parity_manifest.py" + spec = importlib.util.spec_from_file_location("parity_generator", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _registry(generator, facts: list[dict], **options) -> list[str]: + registry = generator.compile_us_fiscal_target_registry( + facts, + target_period=generator.TARGET_PERIOD, + congressional_district_vintage_crosswalk=( + generator.load_congressional_district_vintage_crosswalk( + generator.default_congressional_district_vintage_crosswalk_path() + ) + ), + **options, + ) + registry, _ = generator.apply_us_medicaid_enrollment_substitutions(registry) + rows = [] + for spec in registry.specs: + fields = ( + dataclasses.asdict(spec) if dataclasses.is_dataclass(spec) else vars(spec) + ) + rows.append(json.dumps(fields, default=str, sort_keys=True)) + return rows + + +def main() -> None: + previous_path, new_path, report_path = (Path(arg) for arg in sys.argv[1:4]) + generator = _generator() + previous_facts, _ = generator._load_feed(previous_path) + new_facts, _ = generator._load_feed(new_path) + report: dict[str, dict] = {} + differs = False + for label, options in CONFIGURATIONS.items(): + previous = _registry(generator, previous_facts, **options) + new = _registry(generator, new_facts, **options) + names = [json.loads(row)["name"] for row in new] + first_difference = next( + (i for i, (a, b) in enumerate(zip(previous, new, strict=False)) if a != b), + None, + ) + report[label] = { + "options": options, + "previous_targets": len(previous), + "new_targets": len(new), + "distinct_target_names": len(set(names)), + "ordered_lists_identical": previous == new, + "first_differing_index": first_difference, + "registry_sha256": hashlib.sha256("\n".join(new).encode()).hexdigest(), + } + differs = differs or previous != new + Path(report_path).write_text(json.dumps(report, indent=1, sort_keys=True) + "\n") + print(json.dumps(report, indent=1, sort_keys=True)) + if differs: + raise SystemExit("the compiled registry differs between the feeds") + + +if __name__ == "__main__": + main() diff --git a/experiments/us-esi-454/feed-repin/registry_diff.json b/experiments/us-esi-454/feed-repin/registry_diff.json new file mode 100644 index 0000000000..7164417943 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/registry_diff.json @@ -0,0 +1,25 @@ +{ + "aged": { + "distinct_target_names": 33250, + "first_differing_index": null, + "new_targets": 33250, + "options": { + "age_targets": true + }, + "ordered_lists_identical": true, + "previous_targets": 33250, + "registry_sha256": "054491c95be8741268160682b851ece7e3eb3b1f3a33e968abd09ecac835d171" + }, + "unaged": { + "distinct_target_names": 33250, + "first_differing_index": null, + "new_targets": 33250, + "options": { + "age_targets": false, + "allow_unaged_dollar_targets": true + }, + "ordered_lists_identical": true, + "previous_targets": 33250, + "registry_sha256": "a00aebe49fc492d44bc0b69a4ca4999e2ab5cfe6d423014f02df5e72a0ea58e5" + } +} diff --git a/experiments/us-esi-454/feed-repin/value_diff.json b/experiments/us-esi-454/feed-repin/value_diff.json new file mode 100644 index 0000000000..3155fd32c1 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/value_diff.json @@ -0,0 +1,46 @@ +{ + "new": { + "file": "consumer_facts_us_cf9ece2.jsonl", + "rows": 40181, + "sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57" + }, + "previous": { + "file": "consumer_facts_us_51aa40f.jsonl", + "rows": 40178, + "sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac" + }, + "previous_rows_changed": 0, + "previous_rows_missing": 0, + "rows_added": [ + { + "aggregate_fact_key": "ledger.aggregate_fact.v2:45595c3331d5f2486d15838e", + "period": { + "type": "calendar_year", + "value": 2024 + }, + "record_set_id": "bea_nipa.cy2024.private_group_health_insurance", + "source_sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "value": 977034000000 + }, + { + "aggregate_fact_key": "ledger.aggregate_fact.v2:4950df5409fe51cd6d7afba6", + "period": { + "type": "calendar_year", + "value": 2023 + }, + "record_set_id": "bea_nipa.cy2023.private_group_health_insurance", + "source_sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "value": 923195000000 + }, + { + "aggregate_fact_key": "ledger.aggregate_fact.v2:c78f781d8aaf93cad7570d3f", + "period": { + "type": "calendar_year", + "value": 2025 + }, + "record_set_id": "bea_nipa.cy2025.private_group_health_insurance", + "source_sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "value": 1027929000000 + } + ] +} diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json new file mode 100644 index 0000000000..b0e2bdb2c5 --- /dev/null +++ b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json @@ -0,0 +1,632 @@ +{ + "anchor_gate": { + "details": { + "anchor": { + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", + "line": 17, + "published": "2026-09-30", + "row": "line 17, series B4923C, Private group health insurance", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 + } + }, + "anchor_universe_employer_total": 977034000000.0, + "anchor_value": 977034000000.0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "cross_check_ratio": 0.9331747851002865, + "employed_share_band": [ + 0.8, + 0.95 + ], + "employed_share_of_anchor_universe": 0.8824300567320681, + "employer_premium_by_sector": { + "federal": 36040814058.6149, + "private": 663665461554.1658, + "state_and_local": 162457892436.37872 + }, + "employer_premium_mean_per_positive_person": 11641.867469444154, + "employer_premium_positive_persons": 74057205.19599111, + "employer_premium_total": 862164168049.1594, + "other_policyholder_employer_total": 114869831950.84059, + "private_active_cross_check": { + "employer_premium_private_sector": 663665461554.1658, + "meps_ic_private_active_employer_total": 668573745863.8779, + "ratio": 0.9926585745550477 + }, + "relative_error": 0.0, + "relative_tolerance": 0.05, + "scale_factor": 0.9653323196493044, + "time_period": 2024, + "weighted_employed_policyholders": 79790597.17620382, + "weighted_other_policyholders": 13156453.970184255, + "weighted_policyholders": 92947051.14638808 + }, + "failures": [], + "passed": true + }, + "by_vintage": { + "2022": { + "employer_premium_positive_persons": 24684189.26891587, + "employer_premium_total": 289593560021.9647, + "mean_per_positive_person": 11731.945370660484, + "weighted_persons": 108650789.68666667 + }, + "2023": { + "employer_premium_positive_persons": 24692526.63040858, + "employer_premium_total": 286994649804.61224, + "mean_per_positive_person": 11622.733230190439, + "weighted_persons": 108650789.68666664 + }, + "2024": { + "employer_premium_positive_persons": 24680489.296666663, + "employer_premium_total": 285575958222.5826, + "mean_per_positive_person": 11570.919635744516, + "weighted_persons": 108650789.68666667 + } + }, + "inputs": [ + { + "census_archive_sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + "census_person_member": "pppub23.csv", + "census_person_member_sha256": "19b56537e50e7663f954361ef2bb5ce9cef8d9d45f156fe1a69a99b654198ffe", + "h5": "census_cps_2022.h5", + "h5_sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e", + "income_year": 2022 + }, + { + "census_archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "census_person_member": "pppub24.csv", + "census_person_member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "h5": "census_cps_2023.h5", + "h5_sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", + "income_year": 2023 + }, + { + "census_archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "census_person_member": "pppub25.csv", + "census_person_member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "h5": "census_cps_2024.h5", + "h5_sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", + "income_year": 2024 + } + ], + "issue": "PolicyEngine/microcosm#454", + "other_policyholder_sensitivity": { + "as_built_other_priced_as_active": { + "employer_premium_total": 862164168049.1594, + "mean_per_positive_person": 11641.867469444152 + }, + "nhe_anchor_all_on_employed": { + "employer_premium_total": 1047000000000.0, + "mean_per_positive_person": 14137.71958081773 + }, + "nhe_anchor_other_priced_as_active": { + "employer_premium_total": 923904269398.4752, + "mean_per_positive_person": 12475.548691763057 + }, + "other_65_plus_priced_at_half": { + "employer_premium_total": 883087093221.8387, + "mean_per_positive_person": 11924.391298385674 + }, + "other_not_priced_all_anchor_on_employed": { + "employer_premium_total": 977034000000.0001, + "mean_per_positive_person": 13192.963431637698 + }, + "raw_other_65_plus_share_of_other": 0.4030440846499319, + "weighted_other_policyholders_65_plus": 5861452.143127043, + "weighted_other_policyholders_by_pemlr": { + "0": 106424.49450739344, + "1": 1425715.020176738, + "2": 81652.39084999224, + "3": 221253.36609046787, + "4": 770435.206337095, + "5": 7347264.321874038, + "6": 986297.5838004182, + "7": 2217411.586548111 + } + }, + "person_rows": 432523, + "pool_census_person_columns": [ + { + "archive_sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + "columns_added": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT", + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [], + "income_year": 2022, + "joined_person_rows": 146133, + "member": "pppub23.csv", + "member_sha256": "19b56537e50e7663f954361ef2bb5ce9cef8d9d45f156fe1a69a99b654198ffe", + "member_values": { + "A_ENRLW": { + "0": 74477, + "1": 11613, + "2": 60043 + }, + "A_EXPRRP": { + "1": 38440, + "10": 2799, + "11": 132, + "12": 1735, + "13": 4312, + "14": 919, + "2": 18399, + "3": 11887, + "4": 16284, + "5": 43805, + "7": 2792, + "8": 2765, + "9": 1864 + }, + "A_FTPT": { + "0": 134520, + "1": 10164, + "2": 1449 + }, + "NOEMP": { + "0": 72947, + "1": 13820, + "2": 10422, + "3": 4923, + "4": 8424, + "5": 3859, + "6": 31738 + }, + "NOW_CHAMPVA": { + "1": 260, + "2": 145873 + }, + "NOW_GRPFTYP": { + "0": 106295, + "1": 18329, + "2": 21509 + }, + "NOW_GRPFTYP2": { + "0": 106295, + "1": 10677, + "2": 7652, + "3": 21509 + }, + "NOW_HIPAID": { + "0": 106295, + "1": 8470, + "2": 27889, + "3": 3479 + }, + "NOW_IHSFLG": { + "1": 687, + "2": 145446 + }, + "NOW_MCAID": { + "1": 28184, + "2": 117949 + }, + "NOW_MIL": { + "1": 3882, + "2": 142251 + }, + "NOW_NONM": { + "1": 8666, + "2": 137467 + }, + "NOW_OTHMT": { + "1": 275, + "2": 145858 + }, + "NOW_OWNGRP": { + "0": 67360, + "1": 39838, + "2": 38935 + }, + "NOW_VACARE": { + "1": 1271, + "2": 144862 + }, + "PEMLR": { + "0": 29948, + "1": 66156, + "2": 2539, + "3": 408, + "4": 2064, + "5": 22286, + "6": 5332, + "7": 17400 + }, + "PTOTVAL": { + "negative_rows": 66, + "nonzero_rows": 102532 + } + } + }, + { + "archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "columns_added": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT", + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [], + "income_year": 2023, + "joined_person_rows": 144265, + "member": "pppub24.csv", + "member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "member_values": { + "A_ENRLW": { + "0": 72991, + "1": 11620, + "2": 59654 + }, + "A_EXPRRP": { + "1": 37875, + "10": 2763, + "11": 147, + "12": 1752, + "13": 4387, + "14": 968, + "2": 18376, + "3": 11652, + "4": 16147, + "5": 42756, + "7": 2661, + "8": 2854, + "9": 1927 + }, + "A_FTPT": { + "0": 132645, + "1": 10174, + "2": 1446 + }, + "NOEMP": { + "0": 70794, + "1": 13932, + "2": 10549, + "3": 4902, + "4": 8667, + "5": 3820, + "6": 31601 + }, + "NOW_CHAMPVA": { + "1": 335, + "2": 143930 + }, + "NOW_GRPFTYP": { + "0": 105104, + "1": 17779, + "2": 21382 + }, + "NOW_GRPFTYP2": { + "0": 105104, + "1": 10393, + "2": 7386, + "3": 21382 + }, + "NOW_HIPAID": { + "0": 105104, + "1": 7843, + "2": 28047, + "3": 3271 + }, + "NOW_IHSFLG": { + "1": 651, + "2": 143614 + }, + "NOW_MCAID": { + "1": 26367, + "2": 117898 + }, + "NOW_MIL": { + "1": 4318, + "2": 139947 + }, + "NOW_NONM": { + "1": 8435, + "2": 135830 + }, + "NOW_OTHMT": { + "1": 331, + "2": 143934 + }, + "NOW_OWNGRP": { + "0": 67555, + "1": 39161, + "2": 37549 + }, + "NOW_VACARE": { + "1": 1178, + "2": 143087 + }, + "PEMLR": { + "0": 28915, + "1": 66185, + "2": 2326, + "3": 410, + "4": 2225, + "5": 22025, + "6": 5335, + "7": 16844 + }, + "PTOTVAL": { + "negative_rows": 59, + "nonzero_rows": 102389 + } + } + }, + { + "archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "columns_added": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT" + ], + "income_year": 2024, + "joined_person_rows": 142125, + "member": "pppub25.csv", + "member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "member_values": { + "A_ENRLW": { + "0": 72079, + "1": 11445, + "2": 58601 + }, + "A_EXPRRP": { + "1": 37382, + "10": 2659, + "11": 134, + "12": 1734, + "13": 4257, + "14": 844, + "2": 18380, + "3": 11334, + "4": 15918, + "5": 42053, + "7": 2570, + "8": 2876, + "9": 1984 + }, + "A_FTPT": { + "0": 130680, + "1": 10014, + "2": 1431 + }, + "NOEMP": { + "0": 69665, + "1": 13465, + "2": 10406, + "3": 4884, + "4": 8547, + "5": 3763, + "6": 31395 + }, + "NOW_CHAMPVA": { + "1": 368, + "2": 141757 + }, + "NOW_GRPFTYP": { + "0": 103571, + "1": 17404, + "2": 21150 + }, + "NOW_GRPFTYP2": { + "0": 103571, + "1": 10176, + "2": 7228, + "3": 21150 + }, + "NOW_HIPAID": { + "0": 103571, + "1": 7748, + "2": 27378, + "3": 3428 + }, + "NOW_IHSFLG": { + "1": 685, + "2": 141440 + }, + "NOW_MCAID": { + "1": 24844, + "2": 117281 + }, + "NOW_MIL": { + "1": 4413, + "2": 137712 + }, + "NOW_NONM": { + "1": 8486, + "2": 133639 + }, + "NOW_OTHMT": { + "1": 312, + "2": 141813 + }, + "NOW_OWNGRP": { + "0": 66737, + "1": 38554, + "2": 36834 + }, + "NOW_VACARE": { + "1": 1270, + "2": 140855 + }, + "PEMLR": { + "0": 28155, + "1": 65222, + "2": 2214, + "3": 408, + "4": 2422, + "5": 22064, + "6": 5081, + "7": 16559 + }, + "PTOTVAL": { + "negative_rows": 75, + "nonzero_rows": 101163 + } + } + } + ], + "pooled_income_years": [ + 2022, + 2023, + 2024 + ], + "puf_support_clone": { + "anchor_gate": { + "anchor_universe_employer_total": 977034000000.0001, + "employer_premium_total": 862164168049.1594, + "failures": [], + "passed": true + }, + "person_rows": 865046, + "signal_gate": { + "failures": [], + "passed": true + } + }, + "seconds": 37.5, + "seed": 0, + "signal_gate": { + "failures": [], + "passed": true + }, + "summary": { + "anchor": { + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", + "line": 17, + "published": "2026-09-30", + "row": "line 17, series B4923C, Private group health insurance", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 + } + }, + "anchor_universe_employer_total": 977034000000.0, + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "clone_disagreement_source_persons": 0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "employed_share_of_anchor_universe": 0.8824300567320681, + "employer_premium_by_sector": { + "federal": 36040814058.6149, + "private": 663665461554.1658, + "state_and_local": 162457892436.37872 + }, + "employer_premium_by_tier": { + "employee_plus_one": 186341214403.85696, + "family": 366932888233.55176, + "single": 308890065411.75073 + }, + "employer_premium_mean_per_positive_person": 11641.867469444154, + "employer_premium_outside_universe_rows": 0, + "employer_premium_positive_persons": 74057205.19599111, + "employer_premium_positive_share": 0.2272025370134953, + "employer_premium_total": 862164168049.1594, + "employer_premium_where_employer_pays_none_rows": 0, + "employer_premium_without_raw_share_rows": 0, + "meps_ic_private_active_employer_total": { + "2024": 668573745863.8779, + "2025": 714391090330.8774 + }, + "negative_rows": 0, + "nonfinite_rows": 0, + "other_policyholder_employer_total": 114869831950.84059, + "raw_anchor_universe_total": 1012121919169.708, + "raw_employer_share_band": [ + 6000.0, + 20000.0 + ], + "raw_employer_share_mean_per_positive_person": 12059.958246993665, + "raw_employer_share_total": 893126802552.6951, + "raw_over_published_employer_mean": 1.030960994318845, + "scale_factor": 0.9653323196493044, + "scale_factor_spread": 2.220446049250313e-16, + "source_columns_missing": [], + "weighted_employed_policyholders": 79790597.17620382, + "weighted_employed_policyholders_by_payment": { + "all": 15092018.672620341, + "none": 5733391.980212707, + "some": 58965186.52337077 + }, + "weighted_other_policyholders": 13156453.970184255, + "weighted_other_policyholders_with_employer_share": 10825941.266066976, + "weighted_policyholders": 92947051.14638808 + }, + "target_year": 2024 +} diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json new file mode 100644 index 0000000000..f10ee8482d --- /dev/null +++ b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json @@ -0,0 +1,633 @@ +{ + "anchor_gate": { + "details": { + "anchor": { + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", + "line": 17, + "published": "2026-09-30", + "row": "line 17, series B4923C, Private group health insurance", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 + } + }, + "anchor_universe_employer_total": 977034000000.0, + "anchor_value": 977034000000.0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "cross_check_ratio": 0.9331747851002865, + "employed_share_band": [ + 0.8, + 0.95 + ], + "employed_share_of_anchor_universe": 0.8841568448185655, + "employer_premium_by_sector": { + "federal": 35577196284.01511, + "private": 665582927915.0277, + "state_and_local": 162691174521.41943 + }, + "employer_premium_mean_per_positive_person": 11678.587819371325, + "employer_premium_positive_persons": 73968814.73011559, + "employer_premium_total": 863851298720.4623, + "other_policyholder_employer_total": 113182701279.53773, + "private_active_cross_check": { + "employer_premium_private_sector": 665582927915.0277, + "meps_ic_private_active_employer_total": 668573745863.8779, + "ratio": 0.995526569855079 + }, + "relative_error": 0.0, + "relative_tolerance": 0.05, + "scale_factor": 0.9708291597999409, + "time_period": 2024, + "weighted_employed_policyholders": 79689927.57027404, + "weighted_other_policyholders": 12732394.803675905, + "weighted_policyholders": 92422322.37394994 + }, + "failures": [], + "passed": true + }, + "by_vintage": { + "2023": { + "employer_premium_positive_persons": 24692526.63040858, + "employer_premium_total": 288628868075.3078, + "mean_per_positive_person": 11688.915937823243, + "weighted_persons": 108650789.68666664 + }, + "2024": { + "employer_premium_positive_persons": 24680489.296666663, + "employer_premium_total": 287202098113.7495, + "mean_per_positive_person": 11636.807304000205, + "weighted_persons": 108650789.68666667 + }, + "2025": { + "employer_premium_positive_persons": 24595798.803040348, + "employer_premium_total": 288020332531.40497, + "mean_per_positive_person": 11710.143461402931, + "weighted_persons": 108650789.68666668 + } + }, + "inputs": [ + { + "census_archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "census_person_member": "pppub24.csv", + "census_person_member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "h5": "census_cps_2023.h5", + "h5_sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", + "income_year": 2023 + }, + { + "census_archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "census_person_member": "pppub25.csv", + "census_person_member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "h5": "census_cps_2024.h5", + "h5_sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", + "income_year": 2024 + }, + { + "census_archive_sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + "census_person_member": "pppub26.csv", + "census_person_member_sha256": "f78920699bf671cc211cb720ff8c6274392524a22138aedaf9e3954f91ba4334", + "h5": "census_cps_2025.h5", + "h5_sha256": "4c5a32188b6acfbcfbeb9f5d719d3847873887b72ebdbb19bc402c16e0a64e58", + "income_year": 2025 + } + ], + "issue": "PolicyEngine/microcosm#454", + "other_policyholder_sensitivity": { + "as_built_other_priced_as_active": { + "employer_premium_total": 863851298720.4623, + "mean_per_positive_person": 11678.587819371325 + }, + "nhe_anchor_all_on_employed": { + "employer_premium_total": 1047000000000.0, + "mean_per_positive_person": 14154.61372228431 + }, + "nhe_anchor_other_priced_as_active": { + "employer_premium_total": 925712216525.038, + "mean_per_positive_person": 12514.898608320465 + }, + "other_65_plus_priced_at_half": { + "employer_premium_total": 884704524851.2379, + "mean_per_positive_person": 11960.50697958582 + }, + "other_not_priced_all_anchor_on_employed": { + "employer_premium_total": 977034000000.0, + "mean_per_positive_person": 13208.728618470228 + }, + "raw_other_65_plus_share_of_other": 0.4069439742577271, + "weighted_other_policyholders_65_plus": 5730053.275654608, + "weighted_other_policyholders_by_pemlr": { + "0": 96077.96836891444, + "1": 1388904.5982447905, + "2": 77057.91946861018, + "3": 213860.3873739363, + "4": 792436.9515057954, + "5": 7140960.568448426, + "6": 942816.4562687919, + "7": 2080279.9539966392 + } + }, + "person_rows": 421119, + "pool_census_person_columns": [ + { + "archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "columns_added": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT", + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [], + "income_year": 2023, + "joined_person_rows": 144265, + "member": "pppub24.csv", + "member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "member_values": { + "A_ENRLW": { + "0": 72991, + "1": 11620, + "2": 59654 + }, + "A_EXPRRP": { + "1": 37875, + "10": 2763, + "11": 147, + "12": 1752, + "13": 4387, + "14": 968, + "2": 18376, + "3": 11652, + "4": 16147, + "5": 42756, + "7": 2661, + "8": 2854, + "9": 1927 + }, + "A_FTPT": { + "0": 132645, + "1": 10174, + "2": 1446 + }, + "NOEMP": { + "0": 70794, + "1": 13932, + "2": 10549, + "3": 4902, + "4": 8667, + "5": 3820, + "6": 31601 + }, + "NOW_CHAMPVA": { + "1": 335, + "2": 143930 + }, + "NOW_GRPFTYP": { + "0": 105104, + "1": 17779, + "2": 21382 + }, + "NOW_GRPFTYP2": { + "0": 105104, + "1": 10393, + "2": 7386, + "3": 21382 + }, + "NOW_HIPAID": { + "0": 105104, + "1": 7843, + "2": 28047, + "3": 3271 + }, + "NOW_IHSFLG": { + "1": 651, + "2": 143614 + }, + "NOW_MCAID": { + "1": 26367, + "2": 117898 + }, + "NOW_MIL": { + "1": 4318, + "2": 139947 + }, + "NOW_NONM": { + "1": 8435, + "2": 135830 + }, + "NOW_OTHMT": { + "1": 331, + "2": 143934 + }, + "NOW_OWNGRP": { + "0": 67555, + "1": 39161, + "2": 37549 + }, + "NOW_VACARE": { + "1": 1178, + "2": 143087 + }, + "PEMLR": { + "0": 28915, + "1": 66185, + "2": 2326, + "3": 410, + "4": 2225, + "5": 22025, + "6": 5335, + "7": 16844 + }, + "PTOTVAL": { + "negative_rows": 59, + "nonzero_rows": 102389 + } + } + }, + { + "archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "columns_added": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT" + ], + "income_year": 2024, + "joined_person_rows": 142125, + "member": "pppub25.csv", + "member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "member_values": { + "A_ENRLW": { + "0": 72079, + "1": 11445, + "2": 58601 + }, + "A_EXPRRP": { + "1": 37382, + "10": 2659, + "11": 134, + "12": 1734, + "13": 4257, + "14": 844, + "2": 18380, + "3": 11334, + "4": 15918, + "5": 42053, + "7": 2570, + "8": 2876, + "9": 1984 + }, + "A_FTPT": { + "0": 130680, + "1": 10014, + "2": 1431 + }, + "NOEMP": { + "0": 69665, + "1": 13465, + "2": 10406, + "3": 4884, + "4": 8547, + "5": 3763, + "6": 31395 + }, + "NOW_CHAMPVA": { + "1": 368, + "2": 141757 + }, + "NOW_GRPFTYP": { + "0": 103571, + "1": 17404, + "2": 21150 + }, + "NOW_GRPFTYP2": { + "0": 103571, + "1": 10176, + "2": 7228, + "3": 21150 + }, + "NOW_HIPAID": { + "0": 103571, + "1": 7748, + "2": 27378, + "3": 3428 + }, + "NOW_IHSFLG": { + "1": 685, + "2": 141440 + }, + "NOW_MCAID": { + "1": 24844, + "2": 117281 + }, + "NOW_MIL": { + "1": 4413, + "2": 137712 + }, + "NOW_NONM": { + "1": 8486, + "2": 133639 + }, + "NOW_OTHMT": { + "1": 312, + "2": 141813 + }, + "NOW_OWNGRP": { + "0": 66737, + "1": 38554, + "2": 36834 + }, + "NOW_VACARE": { + "1": 1270, + "2": 140855 + }, + "PEMLR": { + "0": 28155, + "1": 65222, + "2": 2214, + "3": 408, + "4": 2422, + "5": 22064, + "6": 5081, + "7": 16559 + }, + "PTOTVAL": { + "negative_rows": 75, + "nonzero_rows": 101163 + } + } + }, + { + "archive_sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + "columns_added": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT" + ], + "income_year": 2025, + "joined_person_rows": 134729, + "member": "pppub26.csv", + "member_sha256": "f78920699bf671cc211cb720ff8c6274392524a22138aedaf9e3954f91ba4334", + "member_values": { + "A_ENRLW": { + "0": 69572, + "1": 10761, + "2": 54396 + }, + "A_EXPRRP": { + "1": 35882, + "10": 2404, + "11": 97, + "12": 1408, + "13": 3816, + "14": 800, + "2": 17462, + "3": 10808, + "4": 15562, + "5": 39531, + "7": 2452, + "8": 2717, + "9": 1790 + }, + "A_FTPT": { + "0": 123968, + "1": 9470, + "2": 1291 + }, + "NOEMP": { + "0": 66679, + "1": 12893, + "2": 9521, + "3": 4664, + "4": 8218, + "5": 3402, + "6": 29352 + }, + "NOW_CHAMPVA": { + "1": 414, + "2": 134315 + }, + "NOW_GRPFTYP": { + "0": 98544, + "1": 16429, + "2": 19756 + }, + "NOW_GRPFTYP2": { + "0": 98544, + "1": 9623, + "2": 6806, + "3": 19756 + }, + "NOW_HIPAID": { + "0": 98544, + "1": 7449, + "2": 25717, + "3": 3019 + }, + "NOW_IHSFLG": { + "1": 661, + "2": 134068 + }, + "NOW_MCAID": { + "1": 22688, + "2": 112041 + }, + "NOW_MIL": { + "1": 3994, + "2": 130735 + }, + "NOW_NONM": { + "1": 8102, + "2": 126627 + }, + "NOW_OTHMT": { + "1": 235, + "2": 134494 + }, + "NOW_OWNGRP": { + "0": 63878, + "1": 36185, + "2": 34666 + }, + "NOW_VACARE": { + "1": 1377, + "2": 133352 + }, + "PEMLR": { + "0": 25970, + "1": 61505, + "2": 1924, + "3": 382, + "4": 2253, + "5": 22471, + "6": 4889, + "7": 15335 + }, + "PTOTVAL": { + "negative_rows": 64, + "nonzero_rows": 96720 + } + } + } + ], + "pooled_income_years": [ + 2023, + 2024, + 2025 + ], + "puf_support_clone": { + "anchor_gate": { + "anchor_universe_employer_total": 977033999999.9999, + "employer_premium_total": 863851298720.4622, + "failures": [], + "passed": true + }, + "person_rows": 842238, + "signal_gate": { + "failures": [], + "passed": true + } + }, + "seconds": 40.8, + "seed": 0, + "signal_gate": { + "failures": [], + "passed": true + }, + "summary": { + "anchor": { + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", + "line": 17, + "published": "2026-09-30", + "row": "line 17, series B4923C, Private group health insurance", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 + } + }, + "anchor_universe_employer_total": 977034000000.0, + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "clone_disagreement_source_persons": 0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "employed_share_of_anchor_universe": 0.8841568448185655, + "employer_premium_by_sector": { + "federal": 35577196284.01511, + "private": 665582927915.0277, + "state_and_local": 162691174521.41943 + }, + "employer_premium_by_tier": { + "employee_plus_one": 184909763896.50455, + "family": 366869026060.08105, + "single": 312072508763.87665 + }, + "employer_premium_mean_per_positive_person": 11678.587819371325, + "employer_premium_outside_universe_rows": 0, + "employer_premium_positive_persons": 73968814.73011559, + "employer_premium_positive_share": 0.2269313609943412, + "employer_premium_total": 863851298720.4623, + "employer_premium_where_employer_pays_none_rows": 0, + "employer_premium_without_raw_share_rows": 0, + "meps_ic_private_active_employer_total": { + "2024": 668573745863.8779, + "2025": 714391090330.8774 + }, + "negative_rows": 0, + "nonfinite_rows": 0, + "other_policyholder_employer_total": 113182701279.53773, + "raw_anchor_universe_total": 1006391279183.8038, + "raw_employer_share_band": [ + 6000.0, + 20000.0 + ], + "raw_employer_share_mean_per_positive_person": 12029.498394730888, + "raw_employer_share_total": 889807738056.072, + "raw_over_published_employer_mean": 1.0307192789751904, + "scale_factor": 0.9708291597999409, + "scale_factor_spread": 2.220446049250313e-16, + "source_columns_missing": [], + "weighted_employed_policyholders": 79689927.57027404, + "weighted_employed_policyholders_by_payment": { + "all": 14922582.431638531, + "none": 5721112.840158437, + "some": 59046232.29847707 + }, + "weighted_other_policyholders": 12732394.803675905, + "weighted_other_policyholders_with_employer_share": 10538098.517560685, + "weighted_policyholders": 92422322.37394994 + }, + "target_year": 2024 +} diff --git a/experiments/us-esi-454/run_stage_on_pinned_pool.py b/experiments/us-esi-454/run_stage_on_pinned_pool.py new file mode 100644 index 0000000000..80c80ed051 --- /dev/null +++ b/experiments/us-esi-454/run_stage_on_pinned_pool.py @@ -0,0 +1,248 @@ +"""Run the meps_esi_premiums stage on a real pinned ASEC pool. + +Pools sha-pinned processed ASEC H5 inputs exactly as the base build does, +restoring the reviewed Census person columns (#720, #454) from the pinned +official person members, then runs the stage, its signal gate, the release +anchor gate (pre-calibration) and the PUF-support clone, and writes the +measured lineage to ``receipts/stage_on_pool__.json``. + + uv run python experiments/us-esi-454/run_stage_on_pinned_pool.py \ + --asec-h5 2023= --asec-h5 2024=<...> \ + --asec-h5 2025=<...> \ + --census-person-dir ~/.cache/microcosm/cps/asec_education + +The default pool is income years 2023-2025 (``docs/us-asec-source-pins.md``); +2022-2024 is the historical Build J/N/P pool. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +from pathlib import Path + +import numpy as np + +from microcosm.build.us_runtime import esi_premiums as esi +from microcosm.build.us_runtime.asec_pool import ( + AsecSource, + build_pooled_asec_unit_frame, +) +from microcosm.build.us_runtime.asec_sources import ASEC_SOURCE_ARTIFACTS +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + us_esi_premiums_summary, + with_us_esi_premium_inputs, +) +from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support +from microcosm.build.us_runtime.spm_role_source import ASEC_SPM_ROLE_SOURCES + +HERE = Path(__file__).resolve().parent + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _jsonable(value): + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, (np.floating, np.integer)): + return value.item() + return value + + +def _by_vintage(frame, years) -> dict[str, dict[str, float]]: + person = frame.table("person") + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + out = {} + for year in years: + mask = person["source_year"].to_numpy() == year + employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy() + holders = mask & (employer > 0) + out[str(year)] = { + "weighted_persons": float(weights[mask].sum()), + "employer_premium_total": float(weights[mask] @ employer[mask]), + "employer_premium_positive_persons": float(weights[holders].sum()), + "mean_per_positive_person": float( + weights[holders] @ employer[holders] / weights[holders].sum() + ), + } + return out + + +def _other_policyholder_sensitivity(frame, target_year: int) -> dict[str, object]: + """How the employed column moves with the price of non-employed coverage. + + No MEPS-IC table prices retiree coverage, so the stage prices every + policyholder outside the column as an active employee. This re-scales + the anchor universe under two alternatives: the 65-and-over policyholders + outside the column priced at half the active cell, and nobody outside the + column priced at all (the whole anchor loaded onto employed workers). It + also restates the as-built column and the all-on-employed reading against + CMS NHE Table 24, the second estimate of the anchor's concept. + """ + + person = esi._person_with_state(frame) + codes, raw, _premium, _contribution = esi._person_raw_shares(person) + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + universe = codes.universe + other = codes.policyholder & ~universe + age = person["A_AGE"].to_numpy(dtype=np.float64) + older = other & (age >= 65) + anchor = float(esi.EMPLOYER_PREMIUM_ANCHOR["values"][str(target_year)]) + nhe = float(esi.EMPLOYER_PREMIUM_CROSS_CHECK["values"][str(target_year)]) + employed = float(weights[universe] @ raw[universe]) + everyone = float(weights @ raw) + older_raw = float(weights[older] @ raw[older]) + positive = float(weights[universe & (raw > 0)].sum()) + pemlr = person["PEMLR"].to_numpy(dtype=np.int64) + + def column(total_raw: float, total_anchor: float = anchor) -> dict[str, float]: + total = total_anchor * employed / total_raw + return { + "employer_premium_total": total, + "mean_per_positive_person": total / positive, + } + + return { + "as_built_other_priced_as_active": column(everyone), + "other_65_plus_priced_at_half": column(everyone - 0.5 * older_raw), + "other_not_priced_all_anchor_on_employed": column(employed), + "nhe_anchor_other_priced_as_active": column(everyone, nhe), + "nhe_anchor_all_on_employed": column(employed, nhe), + "weighted_other_policyholders_by_pemlr": { + str(code): float(weights[other & (pemlr == code)].sum()) + for code in sorted(set(pemlr[other].tolist())) + }, + "weighted_other_policyholders_65_plus": float(weights[older].sum()), + "raw_other_65_plus_share_of_other": older_raw / (everyone - employed), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--asec-h5", + action="append", + required=True, + help="Pinned processed ASEC input as YEAR=PATH; once per pooled year.", + ) + parser.add_argument("--census-person-dir", type=Path, required=True) + parser.add_argument("--target-year", type=int, default=2024) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args() + paths = { + int(year): Path(path).expanduser() + for year, path in (value.split("=", 1) for value in args.asec_h5) + } + years = tuple(sorted(paths)) + started = time.perf_counter() + sources = [] + inputs = [] + for year in years: + artifact = ASEC_SOURCE_ARTIFACTS[year] + h5 = paths[year] + digest = _sha256(h5) + if digest != artifact.sha256: + raise SystemExit( + f"{h5}: sha256 {digest} is not the pinned {artifact.sha256}" + ) + pin = ASEC_SPM_ROLE_SOURCES[year] + member = args.census_person_dir / pin.member + member_digest = _sha256(member) + if member_digest != pin.csv_sha256: + raise SystemExit( + f"{member}: sha256 {member_digest} is not {pin.csv_sha256}" + ) + sources.append(AsecSource(year=year, path=h5, census_person_source=member)) + inputs.append( + { + "income_year": year, + "h5": artifact.filename, + "h5_sha256": digest, + "census_person_member": pin.member, + "census_person_member_sha256": member_digest, + "census_archive_sha256": pin.archive_sha256, + } + ) + frame, metadata = build_pooled_asec_unit_frame( + sources, target_year=args.target_year + ) + staged = with_us_esi_premium_inputs( + frame, seed=args.seed, time_period=args.target_year + ) + signal = us_esi_premiums_signal_gate(staged) + anchor = us_esi_premiums_anchor_gate(staged, time_period=args.target_year) + cloned = clone_us_frame_for_puf_support(staged) + cloned_signal = us_esi_premiums_signal_gate(cloned) + cloned_anchor = us_esi_premiums_anchor_gate(cloned, time_period=args.target_year) + receipt = { + "issue": "PolicyEngine/microcosm#454", + "inputs": inputs, + "seed": args.seed, + "target_year": args.target_year, + "pooled_income_years": list(years), + "pool_census_person_columns": [ + { + key: _jsonable(source["census_person_columns"][key]) + for key in ( + "income_year", + "member", + "member_sha256", + "archive_sha256", + "joined_person_rows", + "columns_added", + "columns_verified_equal", + "member_values", + ) + } + for source in metadata["sources"] + ], + "summary": _jsonable(us_esi_premiums_summary(staged)), + "by_vintage": _by_vintage(staged, years), + "other_policyholder_sensitivity": _other_policyholder_sensitivity( + staged, args.target_year + ), + "signal_gate": {"passed": signal.passed, "failures": list(signal.failures)}, + "anchor_gate": { + "passed": anchor.passed, + "failures": list(anchor.failures), + "details": _jsonable(dict(anchor.details)), + }, + "puf_support_clone": { + "signal_gate": { + "passed": cloned_signal.passed, + "failures": list(cloned_signal.failures), + }, + "anchor_gate": { + "passed": cloned_anchor.passed, + "failures": list(cloned_anchor.failures), + **{ + key: cloned_anchor.details.get(key) + for key in ( + "employer_premium_total", + "anchor_universe_employer_total", + ) + }, + }, + "person_rows": len(cloned.table("person")), + }, + "person_rows": len(staged.table("person")), + "seconds": round(time.perf_counter() - started, 1), + } + out = HERE / "receipts" / f"stage_on_pool_{years[0]}_{years[-1]}.json" + out.write_text(json.dumps(receipt, indent=1, sort_keys=True) + "\n") + print(json.dumps({k: receipt[k] for k in ("signal_gate", "anchor_gate")}, indent=1)) + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 78d4f523e6..f088d8e37f 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -87,6 +87,7 @@ "derive_energy_subsidy", "derive_education_inputs", "derive_eligibility_inputs", + "derive_employer_sponsored_insurance_premiums", "derive_hours_worked", "derive_housing_tenure_inputs", "derive_immigration_status", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 56f39195cf..f86db2dfb8 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_404 +EXPECTED_AUTHORED_FIELD_COUNT = 32_437 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_184 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_217 class FieldUsageError(AssertionError): @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_716, - "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + 1_749, + "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 2a38471603..089be11cfd 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,20 +350,21 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", + "late_resource_semantics": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - # #1178: puf_support's source bytes attest CPS Social Security preservation. - # RNG rules and the ordered QRF targets are unchanged. - "seed_map": "7d03cc07e973165f38781d5932d5fe6d16ca0b1e4dc854e7841460737fd043fb", - "seed_protocol": "37da8f68444677850a97bdf958b7f62b426a60da570b80a002eeba8af4861f68", - "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + # #1178: puf_support's source bytes attest CPS Social Security preservation; + # #454: the meps_esi_premiums stage joins the attested sources. RNG rules + # and the ordered QRF targets are unchanged. + "seed_map": "08698f4440eb31d1e2886036e7fd0ab86d4c6a007deaa650127dc4346a2151e0", + "seed_protocol": "303da98866edb2a87b1dc36d9ff5a0146deb3a68a4f5361416b5794c13ae4cfb", + "source_manifest": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index fcfe7a3a2e..10e8c8a9e0 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1714,6 +1714,115 @@ "seed_from_build_config" ] }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "anchor": { + "type": "object", + "additionalProperties": false, + "properties": { + "row": { + "type": "string" + }, + "sha256": { + "$ref": "defs.schema.json#/$defs/sha256" + }, + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "table": { + "type": "string" + }, + "values": { + "type": "object", + "additionalProperties": { + "type": "number" + } + } + }, + "required": [ + "row", + "sha256", + "source", + "table", + "values" + ] + }, + "anchor_universe": { + "type": "string", + "x-spec-surface": "documentation" + }, + "cells_resource": { + "type": "string" + }, + "cells_sha256": { + "$ref": "defs.schema.json#/$defs/sha256" + }, + "government_cell_aging": { + "type": "string", + "x-spec-surface": "documentation" + }, + "kind": { + "const": "derive_employer_sponsored_insurance_premiums" + }, + "no_contribution_share": { + "type": "string", + "x-spec-surface": "documentation" + }, + "other_policyholder_cell": { + "type": "string", + "x-spec-surface": "documentation" + }, + "payment_column": { + "type": "string" + }, + "payment_rule": { + "type": "string", + "x-spec-surface": "documentation" + }, + "scaling": { + "type": "string", + "x-spec-surface": "documentation" + }, + "sector_column": { + "type": "string" + }, + "size_column": { + "type": "string" + }, + "suppressed_cell_fallback": { + "type": "string", + "x-spec-surface": "documentation" + }, + "tier_column": { + "type": "string" + }, + "universe": { + "type": "string", + "x-spec-surface": "documentation" + } + }, + "required": [ + "anchor", + "anchor_universe", + "cells_resource", + "cells_sha256", + "government_cell_aging", + "kind", + "no_contribution_share", + "other_policyholder_cell", + "payment_column", + "payment_rule", + "scaling", + "sector_column", + "size_column", + "suppressed_cell_fallback", + "tier_column", + "universe" + ] + }, { "type": "object", "additionalProperties": false, diff --git a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json index 5b46e935e1..0aa420db00 100644 --- a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json +++ b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json @@ -3,15 +3,15 @@ "country": "us", "scope": "us_fiscal_targets", "source_repo": "PolicyEngine/chronicle", - "source_commit": "51aa40fd28e2ca81937752515e773d123fecfb71", - "build": "tools/build_us_chronicle_feed.py --skip-artifact: one `chronicle build-bundle --year Y --source ` per build year in chronicle_feed_scope.json, rows kept for exactly the scope's (record set, period) pairs and sorted by aggregate_fact_key. The commit is c5e5bf8 plus only the TY2023 Historic Table 2 state AGI-band package (tag microcosm-us-feed-940-v1, chronicle#291). Bare feed: `chronicle build-consumer-artifact` refuses it at this commit; see docs/us-chronicle-feed-repin.md.", + "source_commit": "cf9ece2e7be11158c21fdc4dd38b44d92db552f6", + "build": "tools/build_us_chronicle_feed.py --skip-artifact: one `chronicle build-bundle --year Y --source ` per build year in chronicle_feed_scope.json, rows kept for exactly the scope's (record set, period) pairs and sorted by aggregate_fact_key. The commit is c5e5bf8 plus the TY2023 Historic Table 2 state AGI-band package (51aa40f, tag microcosm-us-feed-940-v1, chronicle#291), the digit-text guard match from chronicle main's aa3e4de3, and the BEA NIPA Table 7.8 line 17 (B4923C) package as chronicle#331 merged it (branch feed/us-51aa40f-esi-b4923c, tag microcosm-us-feed-454-v1, microcosm#454). Bare feed: `chronicle build-consumer-artifact` refuses it at this commit; see docs/us-chronicle-feed-repin.md.", "artifact_schema_version": null, "consumer_fact_schema_versions": [ "chronicle.consumer_fact.v3" ], "consumer_fact_schema_sha256": "bdb51e2a8115634633ba7448c4005930fd9c0bfbade5e1b079b6bc24da485d3d", - "fact_row_count": 40178, - "facts_sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac", + "fact_row_count": 40181, + "facts_sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57", "manifest_sha256": null, - "scope_sha256": "f090102099b3d2c799a8f975a8156101dbb18925b526c3791a785f68944d020d" + "scope_sha256": "f4a218fc3be534f3e447f25bf1ec4064d9f3db6b72a55874ed5db4823c922239" } diff --git a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json index d3647ec2b6..596954435f 100644 --- a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json +++ b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json @@ -2,9 +2,9 @@ "version": 1, "country": "us", "source_repo": "PolicyEngine/chronicle", - "source_commit": "51aa40fd28e2ca81937752515e773d123fecfb71", - "rule": "The (record_set_id, period) pairs of the pinned feed consumer_facts_buildn_v9_4.jsonl, plus the 51 TY2023 Historic Table 2 state AGI-band record sets (irs_soi.ty2023.historic_table_2.state_agi., microcosm#940); each built from the named package with `chronicle build-bundle --year --source `.", - "pair_count": 637, + "source_commit": "cf9ece2e7be11158c21fdc4dd38b44d92db552f6", + "rule": "The (record_set_id, period) pairs of the pinned feed consumer_facts_buildn_v9_4.jsonl, plus the 51 TY2023 Historic Table 2 state AGI-band record sets (irs_soi.ty2023.historic_table_2.state_agi., microcosm#940), plus the three BEA NIPA Table 7.8 line 17 (B4923C) record sets (bea_nipa.cy.private_group_health_insurance, 2023 to 2025, microcosm#454); each built from the named package with `chronicle build-bundle --year --source `.", + "pair_count": 640, "runs": { "2020": [ "packages/bea/nipa_total_wages_salaries", @@ -28,6 +28,7 @@ "packages/bea/nipa_pension_contributions", "packages/bea/nipa_personal_income_components", "packages/bea/nipa_personal_income_disposition", + "packages/bea/nipa_private_group_health_insurance", "packages/bea/nipa_total_wages_salaries", "packages/bea/regional_personal_income_state", "packages/cbo/individual_income_tax_receipts_2026_02", @@ -54,6 +55,7 @@ "packages/irs_soi/w2_statistics_2020" ], "2024": [ + "packages/bea/nipa_private_group_health_insurance", "packages/bea/nipa_total_wages_salaries", "packages/cbo/revenue_projections_income_by_source_2026_02", "packages/census/pep_2024_national_age_sex", @@ -74,6 +76,7 @@ "packages/usda_snap/fy69_to_current" ], "2025": [ + "packages/bea/nipa_private_group_health_insurance", "packages/cbo/revenue_projections_income_by_source_2026_02" ], "2026": [ @@ -259,6 +262,14 @@ "package": "packages/bea/nipa_personal_income_disposition", "build_year": 2023 }, + { + "record_set_id": "bea_nipa.cy2023.private_group_health_insurance", + "period_type": "calendar_year", + "period_value": "2023", + "package_id": "bea-nipa-private-group-health-insurance", + "package": "packages/bea/nipa_private_group_health_insurance", + "build_year": 2023 + }, { "record_set_id": "bea_nipa.cy2023.proprietors_income", "period_type": "calendar_year", @@ -315,6 +326,14 @@ "package": "packages/bea/nipa_personal_income_components", "build_year": 2023 }, + { + "record_set_id": "bea_nipa.cy2024.private_group_health_insurance", + "period_type": "calendar_year", + "period_value": "2024", + "package_id": "bea-nipa-private-group-health-insurance", + "package": "packages/bea/nipa_private_group_health_insurance", + "build_year": 2024 + }, { "record_set_id": "bea_nipa.cy2024.total_wages_salaries", "period_type": "calendar_year", @@ -323,6 +342,14 @@ "package": "packages/bea/nipa_total_wages_salaries", "build_year": 2024 }, + { + "record_set_id": "bea_nipa.cy2025.private_group_health_insurance", + "period_type": "calendar_year", + "period_value": "2025", + "package_id": "bea-nipa-private-group-health-insurance", + "package": "packages/bea/nipa_private_group_health_insurance", + "build_year": 2025 + }, { "record_set_id": "bea_regional.cy2023.state_contributions_for_government_social_insurance", "period_type": "calendar_year", diff --git a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json index 97c3bda958..cd769fa333 100644 --- a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json @@ -2,47 +2,6 @@ "schema_version": 1, "description": "Reason'd, issue-linked parity exemption register (the debt ledger). Each entry names a layer the incumbent eCPS populates that the current Microcosm candidate does not, with the reason it is absent and the tracking issue that owns closing it. parity_gate is run with known_gaps = the keys of this file, and fails on any entry whose layer the candidate now populates (stale exemptions cannot rot silently); the launch condition is that this register shrinks to deliberate scope decisions, not TODOs. Seeded from the first parity run of populace_us_2024.h5 against the pinned reference (see ecps_parity_reference.json). Every issue ref is an existing open PolicyEngine/microcosm issue.", "known_gaps": { - "employer_sponsored_insurance_premiums": { - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 197-271 and 1575-1581 requires NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL; datasets/cps/census_cps.py lines 13-55 declares the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs consumed by the hermetic --asec-h5 build contain PHIP_VAL but omit all three required NOW_* fields, so the retired owner/payment/plan-type derivation cannot execute. has_esi and NOW_GRP are not semantically valid substitutes.", - "issue": "PolicyEngine/microcosm#32", - "evidence": { - "classification": "source_unavailability", - "retired_derivation": { - "repository_owner": "PolicyEngine", - "repository_name_parts": ["policyengine-", "us-data"], - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "cps", "cps.py"], - "lines": "197-271,1575-1581" - }, - "optional_source_fields": { - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "cps", "census_cps.py"], - "lines": "13-55" - }, - "required_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "PHIP_VAL"], - "hermetic_inputs": [ - { - "filename": "census_cps_2022.h5", - "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e", - "present_columns": ["PHIP_VAL"], - "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] - }, - { - "filename": "census_cps_2023.h5", - "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", - "present_columns": ["PHIP_VAL"], - "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] - }, - { - "filename": "census_cps_2024.h5", - "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", - "present_columns": ["PHIP_VAL"], - "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] - } - ], - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88, 647-687, and 748-755 consumes those mappings and does not refresh omitted raw Census columns." - } - }, "financial_assistance": { "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1493-1496 derives person-level financial_assistance directly from FIN_VAL; datasets/cps/census_cps.py lines 39-58 and 306-359 make FIN_VAL a required person source. All three SHA-recorded 2022-2024 ASEC H5 inputs consumed by the hermetic --asec-h5 build omit FIN_VAL, FIN_YN, and I_FINVAL from their person tables. HFINVAL and FFINVAL are household/family aggregates and cannot recover the person recipient without synthetic allocation. The retired PUF-clone QRF at datasets/cps/extended_cps.py lines 140-194 and 639-745 trains financial_assistance from CPS person observations, so PUF provides no independent replacement source.", "issue": "PolicyEngine/microcosm#38", diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 0e8c171624..6555a7a3e2 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -72,9 +72,8 @@ "status": "required" }, "employer_sponsored_insurance_premiums": { - "issue": "PolicyEngine/microcosm#32", - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 197-271 and 1575-1581 requires NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL; datasets/cps/census_cps.py lines 13-55 declares the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs consumed by the hermetic --asec-h5 build contain PHIP_VAL but omit all three required NOW_* fields, so the retired owner/payment/plan-type derivation cannot execute. has_esi and NOW_GRP are not semantically valid substitutes.", - "status": "reviewed_exclusion" + "note": "Employer share of the premium for employed current ESI policyholders (PolicyEngine/microcosm#454), written by the meps_esi_premiums base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person files) and MEPS-IC tier x firm-size x State cells, scaled over every policyholder to BEA NIPA 7.8 line 17 (B4923C). A release must persist it: CBO household market income (gov.household.cbo_market_income_additions) and every ESI-to-wages reform bind through it. Required with NO reviewed exclusion; absent on pre-#454 bases by construction — the intended red gate until a base built with the stage ships. The release also holds the anchor-universe total to the anchor (us_esi_premiums_anchor_gate).", + "status": "required" }, "employment_income_before_lsr": { "status": "required" @@ -550,8 +549,8 @@ } }, "counts": { - "required": 167, - "reviewed_exclusion": 7, + "required": 168, + "reviewed_exclusion": 6, "total": 174 }, "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", @@ -567,6 +566,22 @@ "vintage": "2024" }, "reform_coverage_probes": [ + { + "binding_inputs": [ + "employer_sponsored_insurance_premiums" + ], + "budget_measure": "cbo_household_market_income", + "effect_direction": "baseline_minus_reform", + "expected_sign": "positive", + "id": "employer_sponsored_insurance_premium_neutralization", + "issue": "PolicyEngine/microcosm#454", + "min_abs_effect": 500000000000.0, + "name": "Employer-sponsored insurance premium neutralization", + "neutralized_variable": "employer_sponsored_insurance_premiums", + "parameter_changes": {}, + "period": 2024, + "reason": "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums to CBO household market income through gov.household.cbo_market_income_additions, its only consumer, so neutralizing it lowers cbo_household_market_income by the weighted employer-premium total: about $864 billion at 2024 (the employed policyholders' part of BEA NIPA 7.8 line 17's $977.0 billion, the meps_esi_premiums anchor; the stage measured $863.9 billion over 74.0 million weighted employed policyholders on the default 2023-2025 pool, before calibration). The floor sits at about half of that. A structural zero means the column was dropped or the stage never ran." + }, { "binding_inputs": [ "self_employment_income_last_year" diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 72b56c12ab..eee6592225 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1746,46 +1746,107 @@ }, { "stage": "meps_esi_premiums", - "survey": "MEPS-IC", + "survey": "MEPS-IC + CPS ASEC current employment-based coverage", "source": "https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", "grain": "person", "artifacts": [ { "kind": "public_microdata", "format": "census_asec", - "vintage": "hermetic ASEC pool years (2022-2024 for target 2024)", - "locator": "Census CPS ASEC NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL fields; the three NOW_* fields are absent from the current hermetic HDF inputs" + "vintage": "pinned Census ASEC person members, income years 2022-2025", + "locator": "Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM join; every pinned H5 lacks them); PEIO1COW from the pinned H5 inputs; household GESTFIPS as state_fips", + "source_columns": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP", + "PEIO1COW", + "state_fips" + ] }, { "kind": "administrative_table", - "format": "published_table", - "vintage": "build_year", - "locator": "MEPS Insurance Component plan-type premium tables" + "format": "pdf", + "vintage": "2025", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf", + "sha256": "ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", + "size_bytes": 12530005, + "publication": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee), II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent of enrollees whose coverage required no employee contribution), and the enrollment rows II.B.1, II.B.2, II.B.2.b and II.C/D/E.4" + }, + { + "kind": "administrative_table", + "format": "pdf", + "vintage": "2024", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf", + "sha256": "4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", + "size_bytes": 10687163, + "publication": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025 and carry the 2024 private-sector enrollment" + }, + { + "kind": "administrative_table", + "format": "pdf", + "vintage": "2024", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_3/2024/ic24_iiia_g.pdf", + "sha256": "462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79", + "size_bytes": 1765717, + "publication": "MEPS-IC 2024 Series III (State and local government, by government type and size and census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released)." + }, + { + "kind": "administrative_table", + "format": "xlsx", + "vintage": "BEA annual update published 2026-09-30", + "locator": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "size_bytes": 944273, + "measure": "NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025", + "publication": "scale anchor (decision d1317): employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); the pinned US Chronicle feed banks the same values as bea_nipa.cy.private_group_health_insurance" + }, + { + "kind": "administrative_table", + "format": "xlsx", + "vintage": "NHE 2024 release (data through CY2024)", + "locator": "https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "size_bytes": 520391, + "measure": "NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024", + "publication": "cross-check, not gated: a second estimate of employer contributions for active employees, COBRA enrollees and retirees of private, federal and State and local employers, recorded beside the anchor verdict" } ], "operations": [ { "kind": "read_table", - "table": "meps_ic_plan_premiums" + "table": "person", + "weight": "person_weight" }, { - "kind": "assign_by_plan_type", - "inputs": [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - "PHIP_VAL" - ], - "owner_code": 1, - "employer_pays_all_code": 1, - "employer_pays_some_code": 2, - "family_plan_code": 1, - "self_only_plan_code": 2, - "source_unavailable_when_missing": [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP" - ] + "kind": "derive_employer_sponsored_insurance_premiums", + "cells_resource": "microcosm.build.us_runtime.data/meps_ic_esi_premium_cells.json", + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "anchor_universe": "NOW_OWNGRP == 1", + "universe": "NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6)", + "tier_column": "NOW_GRPFTYP2", + "size_column": "NOEMP", + "sector_column": "PEIO1COW", + "payment_column": "NOW_HIPAID", + "payment_rule": "1 (employer paid all): average total premium; 2 (some): average total premium minus average employee contribution / (1 - no-contribution share); 3 (none): 0", + "no_contribution_share": "MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; government cells take the 50-or-more share", + "other_policyholder_cell": "PEIO1COW outside 1-6: the State's all-sizes private cell", + "suppressed_cell_fallback": "national firm-size cell times the State total-to-national ratio", + "government_cell_aging": "MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same tier and measure", + "scaling": "one factor, anchor / weighted raw share of the anchor universe; written to the universe only", + "anchor": { + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "row": "line 17, series B4923C, Private group health insurance", + "values": { + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 + } + } } ], "outputs": [ @@ -1794,7 +1855,7 @@ "nonnegative_outputs": [ "employer_sponsored_insurance_premiums" ], - "notes": "The retired derivation identifies the policy owner with NOW_OWNGRP, employer payment status with NOW_HIPAID, family/self-only plan type with NOW_GRPFTYP, and reported employee-paid premium with PHIP_VAL, then applies MEPS-IC family/self-only total and employee contribution priors (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/cps/cps.py lines 197-271 and 1575-1581). datasets/cps/census_cps.py lines 13-55 declares the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs used by the hermetic build omit the three NOW_* fields, so this stage is intentionally source-unavailable; has_esi and NOW_GRP are forbidden proxies." + "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), the anchor decision d1317 chose; the column carries the employed policyholders' part. CMS NHE Table 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees, which the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band)." }, { "stage": "prior_year_income", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index e33ffec848..3a0bc01f62 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf + sha256: 559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383 stage_manifest: version: 1 country: us @@ -1783,48 +1783,140 @@ stages: per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 overtime-incidence lane.' - stage: meps_esi_premiums - survey: MEPS-IC + survey: MEPS-IC + CPS ASEC current employment-based coverage source: https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp grain: person artifacts: - kind: public_microdata format: census_asec - vintage: hermetic ASEC pool years (2022-2024 for target 2024) - locator: Census CPS ASEC NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL fields; the three NOW_* - fields are absent from the current hermetic HDF inputs - - kind: administrative_table - format: published_table - vintage: build_year - locator: MEPS Insurance Component plan-type premium tables - operations: - - kind: read_table - table: meps_ic_plan_premiums - - kind: assign_by_plan_type - inputs: - - NOW_OWNGRP - - NOW_HIPAID - - NOW_GRPFTYP - - PHIP_VAL - owner_code: 1 - employer_pays_all_code: 1 - employer_pays_some_code: 2 - family_plan_code: 1 - self_only_plan_code: 2 - source_unavailable_when_missing: + vintage: pinned Census ASEC person members, income years 2022-2025 + locator: Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and + NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM + join; every pinned H5 lacks them); PEIO1COW from the pinned H5 inputs; household GESTFIPS as state_fips + source_columns: - NOW_OWNGRP - NOW_HIPAID - NOW_GRPFTYP + - NOW_GRPFTYP2 + - PEMLR + - NOEMP + - PEIO1COW + - state_fips + - kind: administrative_table + format: pdf + vintage: '2025' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf + sha256: ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330 + size_bytes: 12530005 + publication: 'MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average + total premium per enrolled employee), II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent + of enrollees whose coverage required no employee contribution), and the enrollment rows II.B.1, + II.B.2, II.B.2.b and II.C/D/E.4' + - kind: administrative_table + format: pdf + vintage: '2024' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf + sha256: 4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b + size_bytes: 10687163 + publication: MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows + age the 2024 State and local government cells to 2025 and carry the 2024 private-sector enrollment + - kind: administrative_table + format: pdf + vintage: '2024' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_3/2024/ic24_iiia_g.pdf + sha256: 462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79 + size_bytes: 1765717 + publication: 'MEPS-IC 2024 Series III (State and local government, by government type and size and + census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this + series (the 2025 series is not yet released).' + - kind: administrative_table + format: xlsx + vintage: BEA annual update published 2026-09-30 + locator: https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx + sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd + size_bytes: 944273 + measure: NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025 + publication: 'scale anchor (decision d1317): employer contributions for active and retired employees + (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); the pinned US Chronicle + feed banks the same values as bea_nipa.cy.private_group_health_insurance' + - kind: administrative_table + format: xlsx + vintage: NHE 2024 release (data through CY2024) + locator: https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx + sha256: a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420 + size_bytes: 520391 + measure: NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024 + publication: 'cross-check, not gated: a second estimate of employer contributions for active employees, + COBRA enrollees and retirees of private, federal and State and local employers, recorded beside + the anchor verdict' + operations: + - kind: read_table + table: person + weight: person_weight + - kind: derive_employer_sponsored_insurance_premiums + cells_resource: microcosm.build.us_runtime.data/meps_ic_esi_premium_cells.json + cells_sha256: 29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93 + anchor_universe: NOW_OWNGRP == 1 + universe: NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6) + tier_column: NOW_GRPFTYP2 + size_column: NOEMP + sector_column: PEIO1COW + payment_column: NOW_HIPAID + payment_rule: '1 (employer paid all): average total premium; 2 (some): average total premium minus + average employee contribution / (1 - no-contribution share); 3 (none): 0' + no_contribution_share: MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; government + cells take the 50-or-more share + other_policyholder_cell: 'PEIO1COW outside 1-6: the State''s all-sizes private cell' + suppressed_cell_fallback: national firm-size cell times the State total-to-national ratio + government_cell_aging: MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same + tier and measure + scaling: one factor, anchor / weighted raw share of the anchor universe; written to the universe only + anchor: + source: https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx + sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd + table: NIPA Table 7.8 Supplements to Wages and Salaries by Type + row: line 17, series B4923C, Private group health insurance + values: + '2023': 923195000000 + '2024': 977034000000 + '2025': 1027929000000 outputs: - employer_sponsored_insurance_premiums nonnegative_outputs: - employer_sponsored_insurance_premiums - notes: The retired derivation identifies the policy owner with NOW_OWNGRP, employer payment status with - NOW_HIPAID, family/self-only plan type with NOW_GRPFTYP, and reported employee-paid premium with PHIP_VAL, - then applies MEPS-IC family/self-only total and employee contribution priors (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, - datasets/cps/cps.py lines 197-271 and 1575-1581). datasets/cps/census_cps.py lines 13-55 declares - the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs used by the hermetic - build omit the three NOW_* fields, so this stage is intentionally source-unavailable; has_esi and - NOW_GRP are forbidden proxies. + notes: 'Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) + from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI + policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for + active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job + has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed + and workers without pay carry 0, and their scaled share stays out of the column rather than being + loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee + contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State + all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 + family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal + employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells + (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 + ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes + the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the + national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): + premium. Some (2): premium minus the contribution of an employee who pays one, which is the published + average contribution (taken over every enrollee, including those who pay nothing) divided by one minus + the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, + United States row by firm size; State cells of those tables are mostly flagged unreliable; Series + III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes + no share of enrollees who pay the whole premium. One factor scales every policyholder''s share so + the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B + CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), + the anchor decision d1317 chose; the column carries the employed policyholders'' part. CMS NHE Table + 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded + beside the verdict, as is MEPS-IC''s own private-sector enrollment total for active employees, which + the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, + so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities + are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US + treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), + so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, + cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe + total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band).' - stage: prior_year_income survey: CPS ASEC (prior year) source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json index 23e9607ad3..870197d07e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json @@ -1,6 +1,6 @@ { - "feed": "consumer_facts_us_51aa40f.jsonl", - "feed_sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac", + "feed": "consumer_facts_us_cf9ece2.jsonl", + "feed_sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57", "families": { "bea_nipa.business_current_transfer_receipts": 1, "bea_nipa.defined_contribution_actual_contributions": 1, @@ -22,6 +22,7 @@ "bea_nipa.personal_outlays": 1, "bea_nipa.personal_saving": 1, "bea_nipa.personal_saving_rate": 1, + "bea_nipa.private_group_health_insurance": 3, "bea_nipa.proprietors_income": 1, "bea_nipa.rental_income_of_persons": 1, "bea_nipa.social_security_benefits": 1, diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 6de4b30b49..6ff7939935 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -1,15 +1,15 @@ { "schema_version": 1, "reference": { - "feed": "consumer_facts_us_51aa40f.jsonl", - "feed_sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac", + "feed": "consumer_facts_us_cf9ece2.jsonl", + "feed_sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57", "target_period": "2024", "registry_compile": "compile_us_fiscal_target_registry(age_targets=True, canonical CD vintage crosswalk) + apply_us_medicaid_enrollment_substitutions; one national + state + congressional-district surface for every artifact (microcosm#449)", "us_data_source": "retired us-data pipeline (archived): utils/loss.py (eCPS loss matrix), db/etl_national_targets.py, db/etl_*.py, utils/national_target_parity.py", "family_granularity": "namespace.concept of the ledger source_record_id (us_target_family_id)", "governing_rule": "us-data-targeted => compiled unless source-absent; every reviewed exclusion carries a Chesterton's-fence {origin, purpose, verdict_basis}", "compiled_families": "32", - "reviewed_exclusions": "52" + "reviewed_exclusions": "53" }, "families": { "bea_nipa.business_current_transfer_receipts": { @@ -232,6 +232,17 @@ "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." } }, + "bea_nipa.private_group_health_insurance": { + "status": "reviewed_exclusion", + "classification": "deferred", + "reason": "BEA NIPA Table 7.8 line 17 (B4923C), employer contributions for private group health insurance: the scale anchor of the meps_esi_premiums base stage (microcosm#454, decision d1317). The stage prices every current ESI policyholder from MEPS-IC, scales that universe to this fact (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19: contributions for active and retired employees) and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this fact, and the release tool refuses a feed whose values differ from the stage's pinned anchor (us_esi_premiums_anchor_feed_mismatches).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $977.0B at scale factor 0.971; employed column $863.9B (88.4%); other policyholders $113.2B; private-sector part of the column $665.6B against MEPS-IC's own private active-employee total of $668.6B (2024)", + "fence": { + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (this series, the NIPA 7.8 employer-contribution aggregate, before BEA's 2026-09-30 revision; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "purpose": "anchor the CBO market-income ESI component once the column has a live producer. The BEA annual update of 2026-09-30 revised CY2024 from the $1,002.9B the retired target used to $977.034B.", + "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. Whether and how to calibrate the column is decision d1264." + } + }, "bea_nipa.proprietors_income": { "status": "compiled" }, @@ -448,24 +459,24 @@ }, "cms_nhe.esi_employer_contribution_premiums": { "status": "reviewed_exclusion", - "classification": "not_modeled", - "reason": "NHE employer-contribution ESI premium aggregate: a retired us-data calibration target (loss.py 1_002.9e9, removed at 9f320d2c) whose anchor v9.2 re-banks (2026-07-21) for microcosm#454's future gate. The pe-us input it would anchor (employer_sponsored_insurance_premiums, the employer-paid share per the variable's own documentation) has no live producer in the hermetic lineage, so a target would bind against a structural zero. The all-employer series is the future anchor (pe-us covers government and private employees alike); the private-employer subset rides as its cross-check.", - "evidence": "ecps_parity_known_gaps.json entry employer_sponsored_insurance_premiums (NOW_* source-unavailability evidence); microcosm#454 descope comment 2026-07-21", + "classification": "deferred", + "reason": "NHE Table 24 employer contribution to ESI premiums, a second estimate of the concept the meps_esi_premiums stage scales to (microcosm#454): employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper 2024, p. 29). Decision d1317 chose BEA NIPA 7.8 line 17 as the stage's anchor instead; this fact is 7.2% above it for CY2024. It is neither compiled nor gated: us_esi_premiums_anchor_gate records the anchor-universe total against it beside the verdict (cross_check_ratio).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: anchor-universe total $977.0B, 0.933 of NHE's $1,047.0B; scaled to NHE instead, the employed column would be $925.7B", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed facts re-bank that same anchor for the future gate, entered deliberately ahead of the producer.", - "purpose": "anchor a validation gate for the CBO market-income ESI component once the column has a live producer: NIPA 7.8 employer contributions for group health insurance and the NHEA sponsor-of-funds employer share are same-concept estimates (~4 percent apart, CY2024).", - "verdict_basis": "us-data-targeted, so under the absolute rule this compiles UNLESS source-absent \u2014 and the producer source is absent: the declared meps_esi_premiums stage cannot execute because all three required NOW_* fields are missing from the sha-pinned 2022-2024 ASEC h5 inputs (recorded parity known-gap), so a target would bind against a structural zero. Compiles when microcosm#454 restores the source fields or lands a validated PHIP_VAL+MEPS re-derivation; the anchor value is already banked in the feed." + "origin": "not itself a us-data calibration target: the retired pipeline calibrated employer_sponsored_insurance_premiums to the BEA NIPA 7.8 series (family bea_nipa.private_group_health_insurance; utils/loss.py 1_002.9e9 at archived commit 2e044571, removed at 9f320d2c, PR #831). The v9.2 feed banked the NHE Table 24 series for the microcosm#454 gate before decision d1317 moved the stage's anchor to BEA.", + "purpose": "a cross-check on the ESI anchor's level: an independent estimate of the same active-plus-retired concept from the national health accounts.", + "verdict_basis": "deferred, not dropped: the stage's anchor is BEA (d1317), and this fact, like it, includes retirees, so it is not a sum target on the employed column. It is recorded beside the anchor verdict. Whether and how to calibrate the column is decision d1264." } }, "cms_nhe.esi_private_employer_contribution_premiums": { "status": "reviewed_exclusion", - "classification": "not_modeled", - "reason": "NHE employer-contribution ESI premium aggregate: a retired us-data calibration target (loss.py 1_002.9e9, removed at 9f320d2c) whose anchor v9.2 re-banks (2026-07-21) for microcosm#454's future gate. The pe-us input it would anchor (employer_sponsored_insurance_premiums, the employer-paid share per the variable's own documentation) has no live producer in the hermetic lineage, so a target would bind against a structural zero. The all-employer series is the future anchor (pe-us covers government and private employees alike); the private-employer subset rides as its cross-check.", - "evidence": "ecps_parity_known_gaps.json entry employer_sponsored_insurance_premiums (NOW_* source-unavailability evidence); microcosm#454 descope comment 2026-07-21", + "classification": "deferred", + "reason": "NHE Table 24 private-employer contribution to ESI premiums, the private part of the NHE all-employer fact (microcosm#454). It is neither compiled nor gated: the stage scales one factor to BEA NIPA 7.8 line 17 over every current ESI policyholder, and nothing holds the private-sector part of the column to this fact, which also counts private retirees and COBRA enrollees. The private part is instead recorded beside the anchor verdict against MEPS-IC's own active-employee total (us_esi_premiums_anchor_gate, private_active_cross_check).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: private-sector part of the column $665.6B; MEPS-IC private active-employee total $668.6B (2024)", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed facts re-bank that same anchor for the future gate, entered deliberately ahead of the producer.", - "purpose": "anchor a validation gate for the CBO market-income ESI component once the column has a live producer: NIPA 7.8 employer contributions for group health insurance and the NHEA sponsor-of-funds employer share are same-concept estimates (~4 percent apart, CY2024).", - "verdict_basis": "us-data-targeted, so under the absolute rule this compiles UNLESS source-absent \u2014 and the producer source is absent: the declared meps_esi_premiums stage cannot execute because all three required NOW_* fields are missing from the sha-pinned 2022-2024 ASEC h5 inputs (recorded parity known-gap), so a target would bind against a structural zero. Compiles when microcosm#454 restores the source fields or lands a validated PHIP_VAL+MEPS re-derivation; the anchor value is already banked in the feed." + "origin": "not itself a us-data calibration target: the retired pipeline calibrated employer_sponsored_insurance_premiums to the BEA NIPA 7.8 series (family bea_nipa.private_group_health_insurance; utils/loss.py 1_002.9e9 at archived commit 2e044571, removed at 9f320d2c, PR #831). The v9.2 feed banked the NHE Table 24 series for the microcosm#454 gate before decision d1317 moved the stage's anchor to BEA.", + "purpose": "a sector cross-check that private employers' share of the ESI total is in range.", + "verdict_basis": "deferred, not dropped: no gate reads this fact. The private part of the column is compared with MEPS-IC's active-employee total instead, because this fact includes private retirees and COBRA enrollees. Whether and how to calibrate the column is decision d1264." } }, "cms_nhe.medicaid_title_xix_expenditures": { diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 74e4757cba..60a9e52104 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -265,6 +265,21 @@ us_energy_subsidy_summary, with_us_energy_subsidy_input, ) +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, + US_ESI_PREMIUMS_OUTPUT_COLUMNS, + US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, + US_ESI_PREMIUMS_STAGE_NAME, + derive_us_employer_esi_premiums_from_manifest, + load_meps_ic_esi_premium_cells, + us_esi_premiums_anchor_feed_mismatches, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + us_esi_premiums_stage_spec, + us_esi_premiums_summary, + with_us_esi_premium_inputs, +) from microcosm.build.us_runtime.farm_business_income import ( FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL, FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL, @@ -1351,6 +1366,19 @@ "us_energy_subsidy_stage_spec", "us_energy_subsidy_summary", "with_us_energy_subsidy_input", + "US_ESI_EMPLOYER_PREMIUM_COLUMN", + "US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS", + "US_ESI_PREMIUMS_OUTPUT_COLUMNS", + "US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS", + "US_ESI_PREMIUMS_STAGE_NAME", + "derive_us_employer_esi_premiums_from_manifest", + "load_meps_ic_esi_premium_cells", + "us_esi_premiums_anchor_feed_mismatches", + "us_esi_premiums_anchor_gate", + "us_esi_premiums_signal_gate", + "us_esi_premiums_stage_spec", + "us_esi_premiums_summary", + "with_us_esi_premium_inputs", "CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL", "CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL", "CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL", @@ -2135,10 +2163,17 @@ def to_manifest(self) -> dict[str, object]: "structurally impossible under StagePlan." ), ), - "meps_esi_premiums": DonorSpec( - survey="MEPS-IC", + US_ESI_PREMIUMS_STAGE_NAME: DonorSpec( + survey="MEPS-IC + CPS ASEC current employment-based coverage", source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", - notes="Employer-sponsored insurance premium parameters.", + notes=( + "Current ESI policyholders take the MEPS-IC employer share of " + "their tier x firm-size x State (or government census-division) " + "cell, scaled over every policyholder to BEA NIPA 7.8 line 17 " + "(B4923C) and written for the employed ones (microcosm #454). " + "Deterministic; " + "shared by support clones." + ), ), "aca_marketplace_inputs": DonorSpec( survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", @@ -2501,7 +2536,7 @@ def to_manifest(self) -> dict[str, object]: US_SSI_TAKE_UP_STAGE_NAME, "sipp_tips", "org_wages", - "meps_esi_premiums", + US_ESI_PREMIUMS_STAGE_NAME, "mortgage_conversion", "vehicle_assets", US_VOLUNTARY_FILING_STAGE_NAME, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py index 665789e6b6..d0187f65d3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py @@ -61,6 +61,27 @@ The other thirteen columns the 2026-08-23 offline fix appended are not restored; :data:`ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED` records why. + +Microcosm #454 adds six columns that every pinned H5 lacks (2024 included), +read by the ``meps_esi_premiums`` stage (:mod:`.esi_premiums`): + +====================== ====================================================== +Column Build reader +====================== ====================================================== +``NOW_OWNGRP`` ``esi_premiums._esi_person_codes`` -> the current + employment-based policyholder test +``NOW_HIPAID`` same -> employer paid all / some / none of the premium +``NOW_GRPFTYP2`` same -> coverage tier (family / self plus one / + self-only) +``NOW_GRPFTYP`` same -> cross-checked against ``NOW_GRPFTYP2`` +``PEMLR`` same -> employed at interview +``NOEMP`` same -> employer size band +====================== ====================================================== + +Their domains are the codes observed in pppub23/24/25/26 (2026-10-10), equal +to the Census API codebooks for survey years 2023-2026 except that the data +also carry ``PEMLR == 0`` (not in universe: children and Armed Forces), which +the codebooks omit. """ from __future__ import annotations @@ -107,6 +128,8 @@ class AsecCensusPersonColumn: _NOW_YES_NO = frozenset({1, 2}) _ENROLLMENT = frozenset({0, 1, 2}) +_ESI_READER = "microcosm.build.us_runtime.esi_premiums._esi_person_codes" + _HEALTH_COVERAGE_READER = ( "microcosm.build.us_runtime.cps_carried._fill_health_coverage_inputs" ) @@ -118,6 +141,7 @@ class AsecCensusPersonColumn: #: the Census codes observed in the pinned person members pppub23/24/25 #: (2026-09-23); pppub26 (income year 2025) falls inside every one of them #: (2026-09-27). ``A_EXPRRP`` has no code 6 in the Census codebook or the data. +#: The six #454 columns were observed in pppub23/24/25/26 on 2026-10-10. ASEC_CENSUS_PERSON_COLUMNS: tuple[AsecCensusPersonColumn, ...] = ( AsecCensusPersonColumn( "NOW_MCAID", @@ -205,6 +229,38 @@ class AsecCensusPersonColumn: "(is_full_time_college_student <- A_HSCOL == 2 & A_FTPT == 1)", ), ), + # Microcosm #454: the meps_esi_premiums stage's universe, tier, payment + # status, employment and employer-size fields. + AsecCensusPersonColumn( + "NOW_OWNGRP", + frozenset({0, 1, 2}), + (f"{_ESI_READER} (current employment-based policyholder, 1 = yes)",), + ), + AsecCensusPersonColumn( + "NOW_HIPAID", + frozenset({0, 1, 2, 3}), + (f"{_ESI_READER} (employer paid all / some / none of the premium)",), + ), + AsecCensusPersonColumn( + "NOW_GRPFTYP", + frozenset({0, 1, 2}), + (f"{_ESI_READER} (family / self-only, checked against NOW_GRPFTYP2)",), + ), + AsecCensusPersonColumn( + "NOW_GRPFTYP2", + frozenset({0, 1, 2, 3}), + (f"{_ESI_READER} (family / self plus one / self-only coverage tier)",), + ), + AsecCensusPersonColumn( + "PEMLR", + frozenset(range(8)), + (f"{_ESI_READER} (employed at interview: codes 1 and 2)",), + ), + AsecCensusPersonColumn( + "NOEMP", + frozenset(range(7)), + (f"{_ESI_READER} (employer size: under 50 = codes 1-2, 50+ = 3-6)",), + ), ) ASEC_CENSUS_PERSON_COLUMN_NAMES: tuple[str, ...] = tuple( column.name for column in ASEC_CENSUS_PERSON_COLUMNS diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md new file mode 100644 index 0000000000..926a3ae5f3 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md @@ -0,0 +1,45 @@ +# MEPS-IC employer-premium cells + +`meps_ic_esi_premium_cells.json` holds the MEPS-IC averages the +`meps_esi_premiums` stage assigns to ESI policyholders +(`microcosm.build.us_runtime.esi_premiums`, +[PolicyEngine/microcosm#454](https://github.com/PolicyEngine/microcosm/issues/454); +method in `docs/us-esi-employer-premiums.md`). + +AHRQ publishes the State tables only as PDF. Each value is copied as +published; `null` marks a cell AHRQ suppressed (`--`). The fallback for a +suppressed cell is stage logic, not data. + +| Key | Source | Contents | +|---|---|---| +| `private_state_2025` | MEPS-IC 2025 Series II, Tables II.C.1/2, II.D.1/2, II.E.1/2 | Average total premium and average employee contribution per enrolled employee at private-sector establishments, single / family / employee-plus-one, by State, for all firms, under 50 and 50 or more employees | +| `private_national_2024` | MEPS-IC 2024 Series II, United States rows | The same six national averages a year earlier; they age the government cells | +| `public_division_2024` | MEPS-IC 2024 Series III, Tables III.C.1/2, III.D.1/2, III.E.1/2 | The same averages for State and local government jobs, by census division, for all governments and State governments. 2024 is the latest year AHRQ has published for this series | +| `no_contribution_share_2025` | MEPS-IC 2025 Series II, Tables II.C.4.a, II.D.4.a, II.E.4.a, United States row | Percent of enrollees whose coverage required no employee contribution, by tier, for all firms, under 50 and 50 or more employees. National only: most State cells of these tables are flagged unreliable or suppressed | +| `private_enrollment_national` | MEPS-IC 2024 and 2025 Series II, Tables II.B.1, II.B.2, II.B.2.b, II.C.4, II.D.4, II.E.4, United States row | Private-sector employees, the percent in establishments that offer health insurance, the percent enrolled there, and each tier's share of enrollees: the inputs of the active-employee cross-check | +| `state_census_division` | the division headings of Table II.C.1 | State FIPS to census division | + +`sources` pins each PDF by URL, byte length and SHA-256. + +## How it is built + +``` +uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic +``` + +The tool downloads the three pinned PDFs (about 25 MB) into `--pdf-dir`, +refuses any whose length or digest differs from its pin, extracts text with +`pdftotext -layout` (poppler) and reads one row per State, division or +firm-size label. It refuses a label that matches no line or more than one, a +row with the wrong number of values, a table whose title names another survey +year, and a United States row that differs from the AHRQ national figure it +must reproduce (for example single $9,025 and family $26,281 in 2025; the +2024 rows of MEPS-IC Research Findings #54). A national row whose kept column +is suppressed or flagged unreliable is refused too. `--check` regenerates in +memory and fails if any published value, title or pin differs; it ignores the +`pdftotext` version and the `text_line` positions, which depend on the +installed poppler. + +The stage pins this file's own SHA-256 (`esi_premiums._CELLS_SHA256`) and +refuses to load any other bytes, so a regeneration must be re-pinned there +and in `us/source_stages.json`. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json b/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json new file mode 100644 index 0000000000..c13c90de62 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json @@ -0,0 +1,2191 @@ +{ + "description": "MEPS-IC average total premium and average employee contribution per enrolled employee, by coverage tier, for the meps_esi_premiums source stage: private-sector cells by State and firm size (2025), State and local government cells by census division (2024, the latest published year), the 2024 private national rows that age the government cells, the 2025 national share of enrollees whose coverage required no employee contribution, and the 2024 and 2025 national private-sector enrollment rows. Values are copied as published; null marks a cell AHRQ suppressed ('--'). Dollar values are annual; percent values are percents; employee counts are persons.", + "extraction": { + "method": "pdftotext -layout, one row per label, refusing any label that does not match exactly one line", + "pdftotext": "pdftotext version 26.09.0" + }, + "generator": "tools/build_us_meps_ic_esi_cells.py", + "issue": "PolicyEngine/microcosm#454", + "no_contribution_share_2025": { + "employee_plus_one": { + "50plus": 3.6, + "lt50": 23.1, + "text_line": 18025, + "title": "Table II.E.4.a Percent of private-sector employees enrolled in a health insurance plan that take employee-plus-one coverage that required no employee contribution by firm size and State: United States, 2025", + "total": 6.0 + }, + "family": { + "50plus": 4.6, + "lt50": 28.6, + "text_line": 15679, + "title": "Table II.D.4.a Percent of private-sector employees enrolled in a health insurance plan that take family coverage that required no employee contribution by firm size and State: United States, 2025", + "total": 7.7 + }, + "single": { + "50plus": 8.7, + "lt50": 35.5, + "text_line": 11829, + "title": "Table II.C.4.a Percent of private-sector employees enrolled in a health insurance plan that take single coverage that required no employee contribution by firm size and State: United States, 2025", + "total": 13.0 + } + }, + "private_enrollment_national": { + "2024": { + "employees": { + "50plus": 102965293.0, + "lt50": 36475784.0, + "text_line": 3322, + "title": "Table II.B.1 Number of private-sector employees by firm size and State: United States, 2024", + "total": 139441077.0 + }, + "enrolled_percent": { + "50plus": 55.8, + "lt50": 53.0, + "text_line": 4150, + "title": "Table II.B.2.b Percent of private-sector employees that are enrolled in health insurance at establishments that offer health insurance by firm size and State: United States, 2024", + "total": 55.3 + }, + "offer_percent": { + "50plus": 97.4, + "lt50": 50.5, + "text_line": 3650, + "title": "Table II.B.2 Percent of private-sector employees in establishments that offer health insurance by firm size and State: United States, 2024", + "total": 85.1 + }, + "tier_share_percent": { + "employee_plus_one": { + "50plus": 18.1, + "lt50": 14.3, + "text_line": 17851, + "title": "Table II.E.4 Percent of private-sector employees enrolled in a health insurance plan that take employee-plus-one coverage by firm size and State: United States, 2024", + "total": 17.6 + }, + "family": { + "50plus": 24.4, + "lt50": 19.8, + "text_line": 15507, + "title": "Table II.D.4 Percent of private-sector employees enrolled in a health insurance plan that take family coverage by firm size and State: United States, 2024", + "total": 23.7 + }, + "single": { + "50plus": 57.5, + "lt50": 66.0, + "text_line": 11661, + "title": "Table II.C.4 Percent of private-sector employees enrolled in a health insurance plan that take single coverage by firm size and State: United States, 2024", + "total": 58.7 + } + } + }, + "2025": { + "employees": { + "50plus": 105297602.0, + "lt50": 35753890.0, + "text_line": 3330, + "title": "Table II.B.1 Number of private-sector employees by firm size and State: United States, 2025", + "total": 141051492.0 + }, + "enrolled_percent": { + "50plus": 54.7, + "lt50": 52.3, + "text_line": 4156, + "title": "Table II.B.2.b Percent of private-sector employees that are enrolled in health insurance at establishments that offer health insurance by firm size and State: United States, 2025", + "total": 54.3 + }, + "offer_percent": { + "50plus": 95.6, + "lt50": 50.4, + "text_line": 3658, + "title": "Table II.B.2 Percent of private-sector employees in establishments that offer health insurance by firm size and State: United States, 2025", + "total": 84.2 + }, + "tier_share_percent": { + "employee_plus_one": { + "50plus": 18.7, + "lt50": 14.9, + "text_line": 17857, + "title": "Table II.E.4 Percent of private-sector employees enrolled in a health insurance plan that take employee-plus-one coverage by firm size and State: United States, 2025", + "total": 18.1 + }, + "family": { + "50plus": 24.2, + "lt50": 21.6, + "text_line": 15511, + "title": "Table II.D.4 Percent of private-sector employees enrolled in a health insurance plan that take family coverage by firm size and State: United States, 2025", + "total": 23.8 + }, + "single": { + "50plus": 57.1, + "lt50": 63.5, + "text_line": 11663, + "title": "Table II.C.4 Percent of private-sector employees enrolled in a health insurance plan that take single coverage by firm size and State: United States, 2025", + "total": 58.1 + } + } + } + }, + "private_national_2024": { + "employee_plus_one": { + "employee_contribution": { + "text_line": 16845, + "total": 4707.0 + }, + "premium": { + "text_line": 16347, + "total": 16931.0 + } + }, + "family": { + "employee_contribution": { + "text_line": 13493, + "total": 7216.0 + }, + "premium": { + "text_line": 12495, + "total": 24540.0 + } + }, + "single": { + "employee_contribution": { + "text_line": 9645, + "total": 1789.0 + }, + "premium": { + "text_line": 8647, + "total": 8486.0 + } + } + }, + "private_state_2025": { + "employee_plus_one": { + "employee_contribution": { + "rows": { + "01": { + "50plus": 3878.0, + "lt50": 8912.0, + "total": 4545.0 + }, + "02": { + "50plus": 4583.0, + "lt50": null, + "total": 4584.0 + }, + "04": { + "50plus": 4529.0, + "lt50": 3563.0, + "total": 4366.0 + }, + "05": { + "50plus": 5194.0, + "lt50": 3252.0, + "total": 4874.0 + }, + "06": { + "50plus": 4999.0, + "lt50": 6153.0, + "total": 5159.0 + }, + "08": { + "50plus": 4346.0, + "lt50": 6320.0, + "total": 4543.0, + "unreliable": [ + "lt50" + ] + }, + "09": { + "50plus": 3695.0, + "lt50": 5396.0, + "total": 3857.0 + }, + "10": { + "50plus": 5279.0, + "lt50": 3689.0, + "total": 5084.0, + "unreliable": [ + "lt50" + ] + }, + "11": { + "50plus": 5224.0, + "lt50": 3685.0, + "total": 5001.0 + }, + "12": { + "50plus": 4751.0, + "lt50": 4395.0, + "total": 4714.0 + }, + "13": { + "50plus": 4769.0, + "lt50": 5660.0, + "total": 4853.0, + "unreliable": [ + "lt50" + ] + }, + "15": { + "50plus": 4173.0, + "lt50": 2472.0, + "total": 3806.0, + "unreliable": [ + "lt50" + ] + }, + "16": { + "50plus": 4820.0, + "lt50": 6590.0, + "total": 5078.0 + }, + "17": { + "50plus": 5238.0, + "lt50": 3694.0, + "total": 5016.0 + }, + "18": { + "50plus": 4226.0, + "lt50": 6270.0, + "total": 4424.0 + }, + "19": { + "50plus": 4538.0, + "lt50": 3575.0, + "total": 4408.0 + }, + "20": { + "50plus": 4954.0, + "lt50": 4195.0, + "total": 4862.0 + }, + "21": { + "50plus": 3503.0, + "lt50": null, + "total": 3676.0 + }, + "22": { + "50plus": 4682.0, + "lt50": 7519.0, + "total": 5033.0 + }, + "23": { + "50plus": 4353.0, + "lt50": 8221.0, + "total": 4829.0, + "unreliable": [ + "lt50" + ] + }, + "24": { + "50plus": 4821.0, + "lt50": 4989.0, + "total": 4840.0 + }, + "25": { + "50plus": 4785.0, + "lt50": 4715.0, + "total": 4778.0 + }, + "26": { + "50plus": 4232.0, + "lt50": 9482.0, + "total": 5251.0, + "unreliable": [ + "lt50" + ] + }, + "27": { + "50plus": 5760.0, + "lt50": 8314.0, + "total": 6043.0 + }, + "28": { + "50plus": 4317.0, + "lt50": 3552.0, + "total": 4217.0 + }, + "29": { + "50plus": 4795.0, + "lt50": 6982.0, + "total": 4993.0 + }, + "30": { + "50plus": 3495.0, + "lt50": 4138.0, + "total": 3674.0, + "unreliable": [ + "lt50" + ] + }, + "31": { + "50plus": 4593.0, + "lt50": 4432.0, + "total": 4577.0 + }, + "32": { + "50plus": 4324.0, + "lt50": 6257.0, + "total": 4498.0 + }, + "33": { + "50plus": 5072.0, + "lt50": 6280.0, + "total": 5201.0 + }, + "34": { + "50plus": 4675.0, + "lt50": 6836.0, + "total": 5052.0 + }, + "35": { + "50plus": 4317.0, + "lt50": 5360.0, + "total": 4461.0, + "unreliable": [ + "lt50" + ] + }, + "36": { + "50plus": 4604.0, + "lt50": 6460.0, + "total": 4842.0 + }, + "37": { + "50plus": 5068.0, + "lt50": 4148.0, + "total": 4973.0 + }, + "38": { + "50plus": 4921.0, + "lt50": 3347.0, + "total": 4640.0 + }, + "39": { + "50plus": 4081.0, + "lt50": 5839.0, + "total": 4226.0 + }, + "40": { + "50plus": 4768.0, + "lt50": 6016.0, + "total": 4926.0 + }, + "41": { + "50plus": 4743.0, + "lt50": 5794.0, + "total": 4915.0 + }, + "42": { + "50plus": 3846.0, + "lt50": 3916.0, + "total": 3855.0 + }, + "44": { + "50plus": 4813.0, + "lt50": 3981.0, + "total": 4706.0 + }, + "45": { + "50plus": 4919.0, + "lt50": null, + "total": 4879.0 + }, + "46": { + "50plus": 4700.0, + "lt50": null, + "total": 4620.0 + }, + "47": { + "50plus": 4023.0, + "lt50": 5877.0, + "total": 4236.0 + }, + "48": { + "50plus": 4964.0, + "lt50": 8450.0, + "total": 5237.0 + }, + "49": { + "50plus": 3562.0, + "lt50": 4771.0, + "total": 3719.0 + }, + "50": { + "50plus": 4787.0, + "lt50": 7994.0, + "total": 5166.0 + }, + "51": { + "50plus": 4821.0, + "lt50": 6957.0, + "total": 4979.0 + }, + "53": { + "50plus": 4113.0, + "lt50": 5301.0, + "total": 4304.0 + }, + "54": { + "50plus": 3938.0, + "lt50": 5036.0, + "total": 4013.0 + }, + "55": { + "50plus": 4680.0, + "lt50": 5010.0, + "total": 4724.0 + }, + "56": { + "50plus": 4055.0, + "lt50": 6009.0, + "total": 4612.0 + }, + "US": { + "50plus": 4635.0, + "lt50": 5809.0, + "total": 4776.0 + } + }, + "text_line": 16849, + "title": "Table II.E.2 Average total employee contribution (in dollars) per enrolled employee for employee-plus-one coverage at private-sector" + }, + "premium": { + "rows": { + "01": { + "50plus": 16418.0, + "lt50": 14957.0, + "total": 16224.0 + }, + "02": { + "50plus": 20802.0, + "lt50": 20379.0, + "total": 20764.0 + }, + "04": { + "50plus": 15210.0, + "lt50": 11974.0, + "total": 14664.0 + }, + "05": { + "50plus": 15948.0, + "lt50": 14031.0, + "total": 15632.0 + }, + "06": { + "50plus": 18968.0, + "lt50": 17950.0, + "total": 18827.0 + }, + "08": { + "50plus": 17168.0, + "lt50": 19954.0, + "total": 17447.0 + }, + "09": { + "50plus": 19797.0, + "lt50": 16920.0, + "total": 19523.0 + }, + "10": { + "50plus": 18172.0, + "lt50": 16742.0, + "total": 17997.0 + }, + "11": { + "50plus": 19577.0, + "lt50": 19173.0, + "total": 19518.0 + }, + "12": { + "50plus": 17458.0, + "lt50": 17221.0, + "total": 17433.0 + }, + "13": { + "50plus": 17748.0, + "lt50": 22802.0, + "total": 18227.0 + }, + "15": { + "50plus": 17926.0, + "lt50": 15739.0, + "total": 17453.0 + }, + "16": { + "50plus": 17903.0, + "lt50": 16185.0, + "total": 17652.0 + }, + "17": { + "50plus": 19179.0, + "lt50": 20228.0, + "total": 19330.0 + }, + "18": { + "50plus": 15875.0, + "lt50": 15355.0, + "total": 15825.0 + }, + "19": { + "50plus": 17103.0, + "lt50": 14736.0, + "total": 16782.0 + }, + "20": { + "50plus": 17861.0, + "lt50": 15480.0, + "total": 17574.0 + }, + "21": { + "50plus": 15109.0, + "lt50": 18513.0, + "total": 15348.0 + }, + "22": { + "50plus": 17872.0, + "lt50": 18432.0, + "total": 17941.0 + }, + "23": { + "50plus": 18969.0, + "lt50": 18479.0, + "total": 18908.0 + }, + "24": { + "50plus": 18615.0, + "lt50": 16466.0, + "total": 18373.0 + }, + "25": { + "50plus": 20135.0, + "lt50": 18465.0, + "total": 19960.0 + }, + "26": { + "50plus": 16469.0, + "lt50": 17546.0, + "total": 16678.0 + }, + "27": { + "50plus": 18913.0, + "lt50": 16078.0, + "total": 18599.0 + }, + "28": { + "50plus": 14870.0, + "lt50": 14645.0, + "total": 14841.0 + }, + "29": { + "50plus": 17125.0, + "lt50": 16241.0, + "total": 17045.0 + }, + "30": { + "50plus": 15634.0, + "lt50": 16382.0, + "total": 15842.0 + }, + "31": { + "50plus": 17402.0, + "lt50": 15704.0, + "total": 17232.0 + }, + "32": { + "50plus": 14982.0, + "lt50": 13611.0, + "total": 14858.0 + }, + "33": { + "50plus": 20224.0, + "lt50": 17855.0, + "total": 19972.0 + }, + "34": { + "50plus": 19631.0, + "lt50": 19176.0, + "total": 19551.0 + }, + "35": { + "50plus": 19059.0, + "lt50": 21911.0, + "total": 19454.0 + }, + "36": { + "50plus": 20376.0, + "lt50": 23553.0, + "total": 20784.0 + }, + "37": { + "50plus": 16396.0, + "lt50": 12131.0, + "total": 15958.0 + }, + "38": { + "50plus": 18518.0, + "lt50": 13977.0, + "total": 17707.0 + }, + "39": { + "50plus": 18113.0, + "lt50": 16253.0, + "total": 17960.0 + }, + "40": { + "50plus": 16288.0, + "lt50": 14048.0, + "total": 16005.0 + }, + "41": { + "50plus": 18239.0, + "lt50": 16815.0, + "total": 18006.0 + }, + "42": { + "50plus": 19736.0, + "lt50": 17599.0, + "total": 19455.0 + }, + "44": { + "50plus": 20372.0, + "lt50": 16114.0, + "total": 19825.0 + }, + "45": { + "50plus": 16508.0, + "lt50": 14947.0, + "total": 16317.0 + }, + "46": { + "50plus": 18264.0, + "lt50": 14750.0, + "total": 17990.0 + }, + "47": { + "50plus": 16796.0, + "lt50": 16341.0, + "total": 16743.0 + }, + "48": { + "50plus": 16510.0, + "lt50": 18357.0, + "total": 16654.0 + }, + "49": { + "50plus": 16949.0, + "lt50": 14209.0, + "total": 16593.0 + }, + "50": { + "50plus": 18840.0, + "lt50": 21527.0, + "total": 19157.0 + }, + "51": { + "50plus": 17533.0, + "lt50": 16454.0, + "total": 17453.0 + }, + "53": { + "50plus": 18121.0, + "lt50": 17787.0, + "total": 18067.0 + }, + "54": { + "50plus": 18824.0, + "lt50": 13947.0, + "total": 18492.0 + }, + "55": { + "50plus": 19345.0, + "lt50": 19165.0, + "total": 19321.0 + }, + "56": { + "50plus": 19248.0, + "lt50": 16466.0, + "total": 18455.0 + }, + "US": { + "50plus": 17932.0, + "lt50": 17678.0, + "total": 17901.0 + } + }, + "text_line": 16351, + "title": "Table II.E.1 Average total employee-plus-one premium (in dollars) per enrolled employee at private-sector establishments that offer" + } + }, + "family": { + "employee_contribution": { + "rows": { + "01": { + "50plus": 7797.0, + "lt50": 5592.0, + "total": 7540.0 + }, + "02": { + "50plus": 6272.0, + "lt50": null, + "total": 6171.0 + }, + "04": { + "50plus": 8624.0, + "lt50": 7732.0, + "total": 8493.0 + }, + "05": { + "50plus": 8205.0, + "lt50": 5480.0, + "total": 7824.0 + }, + "06": { + "50plus": 7972.0, + "lt50": 7166.0, + "total": 7847.0 + }, + "08": { + "50plus": 6659.0, + "lt50": 7191.0, + "total": 6733.0 + }, + "09": { + "50plus": 4614.0, + "lt50": 6788.0, + "total": 4979.0, + "unreliable": [ + "lt50" + ] + }, + "10": { + "50plus": 7184.0, + "lt50": null, + "total": 6993.0 + }, + "11": { + "50plus": 8191.0, + "lt50": 7114.0, + "total": 8028.0 + }, + "12": { + "50plus": 7676.0, + "lt50": 10263.0, + "total": 7976.0 + }, + "13": { + "50plus": 7379.0, + "lt50": 5677.0, + "total": 7224.0 + }, + "15": { + "50plus": 5310.0, + "lt50": 6819.0, + "total": 5531.0 + }, + "16": { + "50plus": 6506.0, + "lt50": 6135.0, + "total": 6439.0 + }, + "17": { + "50plus": 7911.0, + "lt50": 7377.0, + "total": 7838.0 + }, + "18": { + "50plus": 6808.0, + "lt50": 5322.0, + "total": 6650.0, + "unreliable": [ + "lt50" + ] + }, + "19": { + "50plus": 7102.0, + "lt50": 10963.0, + "total": 7533.0 + }, + "20": { + "50plus": 7908.0, + "lt50": 9353.0, + "total": 8222.0 + }, + "21": { + "50plus": 5858.0, + "lt50": 7769.0, + "total": 5982.0 + }, + "22": { + "50plus": 7191.0, + "lt50": 11197.0, + "total": 7808.0 + }, + "23": { + "50plus": 6581.0, + "lt50": 10518.0, + "total": 7079.0 + }, + "24": { + "50plus": 6107.0, + "lt50": 10129.0, + "total": 6502.0 + }, + "25": { + "50plus": 7392.0, + "lt50": 7943.0, + "total": 7501.0 + }, + "26": { + "50plus": 6199.0, + "lt50": 7727.0, + "total": 6437.0 + }, + "27": { + "50plus": 6622.0, + "lt50": 9801.0, + "total": 7013.0 + }, + "28": { + "50plus": 6021.0, + "lt50": 8694.0, + "total": 6228.0 + }, + "29": { + "50plus": 6662.0, + "lt50": 7909.0, + "total": 6797.0 + }, + "30": { + "50plus": 5282.0, + "lt50": 4365.0, + "total": 4967.0, + "unreliable": [ + "lt50" + ] + }, + "31": { + "50plus": 6641.0, + "lt50": 4775.0, + "total": 6369.0 + }, + "32": { + "50plus": 7953.0, + "lt50": 7528.0, + "total": 7902.0 + }, + "33": { + "50plus": 7926.0, + "lt50": 5703.0, + "total": 7434.0, + "unreliable": [ + "lt50" + ] + }, + "34": { + "50plus": 6745.0, + "lt50": 9849.0, + "total": 7053.0 + }, + "35": { + "50plus": 6586.0, + "lt50": null, + "total": 6761.0 + }, + "36": { + "50plus": 6596.0, + "lt50": 9527.0, + "total": 7075.0 + }, + "37": { + "50plus": 7894.0, + "lt50": 10336.0, + "total": 8073.0 + }, + "38": { + "50plus": 5632.0, + "lt50": 5409.0, + "total": 5586.0 + }, + "39": { + "50plus": 6828.0, + "lt50": 4392.0, + "total": 6453.0, + "unreliable": [ + "lt50" + ] + }, + "40": { + "50plus": 6820.0, + "lt50": 7568.0, + "total": 6955.0 + }, + "41": { + "50plus": 9550.0, + "lt50": 8246.0, + "total": 9374.0 + }, + "42": { + "50plus": 6936.0, + "lt50": 5659.0, + "total": 6720.0 + }, + "44": { + "50plus": 7002.0, + "lt50": 7194.0, + "total": 7028.0 + }, + "45": { + "50plus": 8307.0, + "lt50": null, + "total": 8269.0 + }, + "46": { + "50plus": 6983.0, + "lt50": 6288.0, + "total": 6892.0 + }, + "47": { + "50plus": 5680.0, + "lt50": 9127.0, + "total": 5972.0 + }, + "48": { + "50plus": 7571.0, + "lt50": 11247.0, + "total": 7913.0 + }, + "49": { + "50plus": 5625.0, + "lt50": 5199.0, + "total": 5567.0 + }, + "50": { + "50plus": 7324.0, + "lt50": 11701.0, + "total": 7870.0 + }, + "51": { + "50plus": 9012.0, + "lt50": 6839.0, + "total": 8692.0 + }, + "53": { + "50plus": 8501.0, + "lt50": 5989.0, + "total": 8222.0 + }, + "54": { + "50plus": 4966.0, + "lt50": null, + "total": 5129.0 + }, + "55": { + "50plus": 6698.0, + "lt50": 3464.0, + "total": 6148.0 + }, + "56": { + "50plus": 6666.0, + "lt50": 7410.0, + "total": 6830.0 + }, + "US": { + "50plus": 7246.0, + "lt50": 7760.0, + "total": 7314.0 + } + }, + "text_line": 13495, + "title": "Table II.D.2 Average total employee contribution (in dollars) per enrolled employee for family coverage at private-sector establishments" + }, + "premium": { + "rows": { + "01": { + "50plus": 24093.0, + "lt50": 19467.0, + "total": 23554.0 + }, + "02": { + "50plus": 29538.0, + "lt50": 39824.0, + "total": 30132.0 + }, + "04": { + "50plus": 23190.0, + "lt50": 22869.0, + "total": 23143.0 + }, + "05": { + "50plus": 23777.0, + "lt50": 17908.0, + "total": 22956.0 + }, + "06": { + "50plus": 27727.0, + "lt50": 23389.0, + "total": 27057.0 + }, + "08": { + "50plus": 25668.0, + "lt50": 21275.0, + "total": 25055.0 + }, + "09": { + "50plus": 29669.0, + "lt50": 23892.0, + "total": 28700.0 + }, + "10": { + "50plus": 26070.0, + "lt50": 22354.0, + "total": 25600.0 + }, + "11": { + "50plus": 29886.0, + "lt50": 26474.0, + "total": 29371.0 + }, + "12": { + "50plus": 25454.0, + "lt50": 24957.0, + "total": 25396.0 + }, + "13": { + "50plus": 26308.0, + "lt50": 27747.0, + "total": 26439.0 + }, + "15": { + "50plus": 23278.0, + "lt50": 25498.0, + "total": 23603.0 + }, + "16": { + "50plus": 26821.0, + "lt50": 22091.0, + "total": 25957.0 + }, + "17": { + "50plus": 28288.0, + "lt50": 27996.0, + "total": 28248.0 + }, + "18": { + "50plus": 23868.0, + "lt50": 20362.0, + "total": 23497.0 + }, + "19": { + "50plus": 25195.0, + "lt50": 25071.0, + "total": 25181.0 + }, + "20": { + "50plus": 27038.0, + "lt50": 19915.0, + "total": 25493.0 + }, + "21": { + "50plus": 24775.0, + "lt50": 20839.0, + "total": 24520.0 + }, + "22": { + "50plus": 26099.0, + "lt50": 23163.0, + "total": 25647.0 + }, + "23": { + "50plus": 26617.0, + "lt50": 22577.0, + "total": 26106.0 + }, + "24": { + "50plus": 24670.0, + "lt50": 25156.0, + "total": 24717.0 + }, + "25": { + "50plus": 30238.0, + "lt50": 27129.0, + "total": 29619.0 + }, + "26": { + "50plus": 23913.0, + "lt50": 23191.0, + "total": 23800.0 + }, + "27": { + "50plus": 26446.0, + "lt50": 22393.0, + "total": 25947.0 + }, + "28": { + "50plus": 22249.0, + "lt50": 18158.0, + "total": 21932.0 + }, + "29": { + "50plus": 25723.0, + "lt50": 23850.0, + "total": 25520.0 + }, + "30": { + "50plus": 22878.0, + "lt50": 20292.0, + "total": 21989.0 + }, + "31": { + "50plus": 26754.0, + "lt50": 27269.0, + "total": 26830.0 + }, + "32": { + "50plus": 23408.0, + "lt50": 20601.0, + "total": 23075.0 + }, + "33": { + "50plus": 29510.0, + "lt50": 23407.0, + "total": 28161.0 + }, + "34": { + "50plus": 30043.0, + "lt50": 26238.0, + "total": 29666.0 + }, + "35": { + "50plus": 26403.0, + "lt50": 27361.0, + "total": 26482.0 + }, + "36": { + "50plus": 29635.0, + "lt50": 30351.0, + "total": 29752.0 + }, + "37": { + "50plus": 25467.0, + "lt50": 23343.0, + "total": 25311.0 + }, + "38": { + "50plus": 25153.0, + "lt50": 23944.0, + "total": 24903.0 + }, + "39": { + "50plus": 25377.0, + "lt50": 20726.0, + "total": 24661.0 + }, + "40": { + "50plus": 24628.0, + "lt50": 23828.0, + "total": 24483.0 + }, + "41": { + "50plus": 28022.0, + "lt50": 21815.0, + "total": 27182.0 + }, + "42": { + "50plus": 27077.0, + "lt50": 24389.0, + "total": 26623.0 + }, + "44": { + "50plus": 28020.0, + "lt50": 23044.0, + "total": 27361.0 + }, + "45": { + "50plus": 25209.0, + "lt50": 23531.0, + "total": 25046.0 + }, + "46": { + "50plus": 26661.0, + "lt50": 22476.0, + "total": 26110.0 + }, + "47": { + "50plus": 25166.0, + "lt50": 24654.0, + "total": 25123.0 + }, + "48": { + "50plus": 25371.0, + "lt50": 21949.0, + "total": 25053.0 + }, + "49": { + "50plus": 25736.0, + "lt50": 21266.0, + "total": 25122.0 + }, + "50": { + "50plus": 25016.0, + "lt50": 28494.0, + "total": 25450.0 + }, + "51": { + "50plus": 28538.0, + "lt50": 19835.0, + "total": 27257.0 + }, + "53": { + "50plus": 28199.0, + "lt50": 22978.0, + "total": 27618.0 + }, + "54": { + "50plus": 26213.0, + "lt50": 24948.0, + "total": 26094.0 + }, + "55": { + "50plus": 27093.0, + "lt50": 28362.0, + "total": 27309.0 + }, + "56": { + "50plus": 27668.0, + "lt50": 25028.0, + "total": 27084.0 + }, + "US": { + "50plus": 26602.0, + "lt50": 24176.0, + "total": 26281.0 + } + }, + "text_line": 12499, + "title": "Table II.D.1 Average total family premium (in dollars) per enrolled employee at private-sector establishments that offer health" + } + }, + "single": { + "employee_contribution": { + "rows": { + "01": { + "50plus": 2005.0, + "lt50": 2336.0, + "total": 2048.0 + }, + "02": { + "50plus": 1945.0, + "lt50": 1866.0, + "total": 1931.0 + }, + "04": { + "50plus": 1722.0, + "lt50": 2503.0, + "total": 1837.0, + "unreliable": [ + "lt50" + ] + }, + "05": { + "50plus": 1859.0, + "lt50": 1404.0, + "total": 1795.0 + }, + "06": { + "50plus": 1499.0, + "lt50": 1602.0, + "total": 1515.0 + }, + "08": { + "50plus": 1685.0, + "lt50": 1295.0, + "total": 1618.0 + }, + "09": { + "50plus": 1755.0, + "lt50": 1843.0, + "total": 1771.0 + }, + "10": { + "50plus": 2179.0, + "lt50": 1806.0, + "total": 2120.0, + "unreliable": [ + "lt50" + ] + }, + "11": { + "50plus": 1885.0, + "lt50": 1777.0, + "total": 1863.0 + }, + "12": { + "50plus": 1825.0, + "lt50": 2500.0, + "total": 1913.0 + }, + "13": { + "50plus": 1710.0, + "lt50": 1994.0, + "total": 1748.0 + }, + "15": { + "50plus": 1040.0, + "lt50": 404.0, + "total": 857.0, + "unreliable": [ + "lt50" + ] + }, + "16": { + "50plus": 1733.0, + "lt50": 1477.0, + "total": 1686.0 + }, + "17": { + "50plus": 1820.0, + "lt50": 2112.0, + "total": 1858.0 + }, + "18": { + "50plus": 2158.0, + "lt50": 1920.0, + "total": 2137.0 + }, + "19": { + "50plus": 2003.0, + "lt50": 2150.0, + "total": 2025.0 + }, + "20": { + "50plus": 1796.0, + "lt50": 1724.0, + "total": 1784.0 + }, + "21": { + "50plus": 1884.0, + "lt50": 2306.0, + "total": 1947.0, + "unreliable": [ + "lt50" + ] + }, + "22": { + "50plus": 2123.0, + "lt50": 2727.0, + "total": 2259.0 + }, + "23": { + "50plus": 1846.0, + "lt50": 1963.0, + "total": 1869.0 + }, + "24": { + "50plus": 2193.0, + "lt50": 1586.0, + "total": 2065.0 + }, + "25": { + "50plus": 1930.0, + "lt50": 2497.0, + "total": 2038.0 + }, + "26": { + "50plus": 1945.0, + "lt50": 2075.0, + "total": 1970.0 + }, + "27": { + "50plus": 1770.0, + "lt50": 2661.0, + "total": 1939.0 + }, + "28": { + "50plus": 1743.0, + "lt50": 1566.0, + "total": 1716.0 + }, + "29": { + "50plus": 1645.0, + "lt50": 1671.0, + "total": 1649.0 + }, + "30": { + "50plus": 1343.0, + "lt50": 1799.0, + "total": 1493.0 + }, + "31": { + "50plus": 1602.0, + "lt50": 2206.0, + "total": 1722.0 + }, + "32": { + "50plus": 1586.0, + "lt50": 1422.0, + "total": 1563.0 + }, + "33": { + "50plus": 2335.0, + "lt50": 2410.0, + "total": 2350.0 + }, + "34": { + "50plus": 2088.0, + "lt50": 2698.0, + "total": 2223.0 + }, + "35": { + "50plus": 1682.0, + "lt50": 2171.0, + "total": 1767.0, + "unreliable": [ + "lt50" + ] + }, + "36": { + "50plus": 2119.0, + "lt50": 2885.0, + "total": 2231.0 + }, + "37": { + "50plus": 2013.0, + "lt50": 2012.0, + "total": 2012.0 + }, + "38": { + "50plus": 1737.0, + "lt50": 1179.0, + "total": 1579.0 + }, + "39": { + "50plus": 1985.0, + "lt50": 2206.0, + "total": 2016.0 + }, + "40": { + "50plus": 1517.0, + "lt50": 1750.0, + "total": 1558.0 + }, + "41": { + "50plus": 1514.0, + "lt50": 1448.0, + "total": 1499.0 + }, + "42": { + "50plus": 1645.0, + "lt50": 1774.0, + "total": 1669.0 + }, + "44": { + "50plus": 2082.0, + "lt50": 1902.0, + "total": 2046.0 + }, + "45": { + "50plus": 1804.0, + "lt50": 1881.0, + "total": 1814.0 + }, + "46": { + "50plus": 1665.0, + "lt50": 2002.0, + "total": 1730.0 + }, + "47": { + "50plus": 1743.0, + "lt50": 986.0, + "total": 1645.0 + }, + "48": { + "50plus": 1646.0, + "lt50": 1583.0, + "total": 1638.0 + }, + "49": { + "50plus": 1621.0, + "lt50": 1328.0, + "total": 1591.0 + }, + "50": { + "50plus": 2022.0, + "lt50": 2622.0, + "total": 2187.0 + }, + "51": { + "50plus": 2120.0, + "lt50": 1763.0, + "total": 2054.0 + }, + "53": { + "50plus": 1462.0, + "lt50": 1088.0, + "total": 1381.0 + }, + "54": { + "50plus": 2118.0, + "lt50": 1388.0, + "total": 2012.0, + "unreliable": [ + "lt50" + ] + }, + "55": { + "50plus": 2102.0, + "lt50": 1809.0, + "total": 2052.0 + }, + "56": { + "50plus": 1620.0, + "lt50": 1258.0, + "total": 1497.0, + "unreliable": [ + "lt50" + ] + }, + "US": { + "50plus": 1796.0, + "lt50": 1924.0, + "total": 1817.0 + } + }, + "text_line": 9647, + "title": "Table II.C.2 Average total employee contribution (in dollars) per enrolled employee for single coverage at private-sector establishments" + }, + "premium": { + "rows": { + "01": { + "50plus": 8216.0, + "lt50": 7854.0, + "total": 8169.0 + }, + "02": { + "50plus": 10536.0, + "lt50": 11082.0, + "total": 10630.0 + }, + "04": { + "50plus": 7578.0, + "lt50": 8640.0, + "total": 7734.0 + }, + "05": { + "50plus": 7823.0, + "lt50": 7691.0, + "total": 7804.0 + }, + "06": { + "50plus": 9164.0, + "lt50": 9149.0, + "total": 9162.0 + }, + "08": { + "50plus": 8977.0, + "lt50": 8039.0, + "total": 8816.0 + }, + "09": { + "50plus": 10364.0, + "lt50": 8243.0, + "total": 9987.0 + }, + "10": { + "50plus": 9247.0, + "lt50": 10589.0, + "total": 9459.0 + }, + "11": { + "50plus": 9895.0, + "lt50": 9252.0, + "total": 9762.0 + }, + "12": { + "50plus": 8795.0, + "lt50": 9742.0, + "total": 8919.0 + }, + "13": { + "50plus": 8768.0, + "lt50": 9801.0, + "total": 8907.0 + }, + "15": { + "50plus": 8250.0, + "lt50": 8372.0, + "total": 8285.0 + }, + "16": { + "50plus": 8760.0, + "lt50": 8815.0, + "total": 8770.0 + }, + "17": { + "50plus": 9703.0, + "lt50": 9865.0, + "total": 9724.0 + }, + "18": { + "50plus": 8455.0, + "lt50": 7330.0, + "total": 8355.0 + }, + "19": { + "50plus": 8629.0, + "lt50": 8572.0, + "total": 8620.0 + }, + "20": { + "50plus": 8874.0, + "lt50": 7572.0, + "total": 8657.0 + }, + "21": { + "50plus": 8810.0, + "lt50": 9644.0, + "total": 8936.0 + }, + "22": { + "50plus": 8624.0, + "lt50": 9429.0, + "total": 8806.0 + }, + "23": { + "50plus": 9983.0, + "lt50": 9062.0, + "total": 9803.0 + }, + "24": { + "50plus": 9061.0, + "lt50": 7628.0, + "total": 8760.0 + }, + "25": { + "50plus": 10548.0, + "lt50": 9965.0, + "total": 10436.0 + }, + "26": { + "50plus": 8193.0, + "lt50": 8384.0, + "total": 8230.0 + }, + "27": { + "50plus": 8644.0, + "lt50": 8336.0, + "total": 8586.0 + }, + "28": { + "50plus": 7637.0, + "lt50": 7477.0, + "total": 7612.0 + }, + "29": { + "50plus": 8039.0, + "lt50": 8370.0, + "total": 8084.0 + }, + "30": { + "50plus": 8826.0, + "lt50": 9049.0, + "total": 8899.0 + }, + "31": { + "50plus": 8822.0, + "lt50": 9253.0, + "total": 8907.0 + }, + "32": { + "50plus": 8213.0, + "lt50": 8232.0, + "total": 8215.0 + }, + "33": { + "50plus": 9768.0, + "lt50": 8505.0, + "total": 9502.0 + }, + "34": { + "50plus": 9552.0, + "lt50": 10415.0, + "total": 9743.0 + }, + "35": { + "50plus": 9445.0, + "lt50": 9459.0, + "total": 9447.0 + }, + "36": { + "50plus": 10760.0, + "lt50": 11804.0, + "total": 10913.0 + }, + "37": { + "50plus": 8728.0, + "lt50": 7872.0, + "total": 8612.0 + }, + "38": { + "50plus": 9349.0, + "lt50": 9238.0, + "total": 9318.0 + }, + "39": { + "50plus": 8872.0, + "lt50": 8379.0, + "total": 8804.0 + }, + "40": { + "50plus": 8101.0, + "lt50": 8231.0, + "total": 8124.0 + }, + "41": { + "50plus": 9407.0, + "lt50": 8594.0, + "total": 9225.0 + }, + "42": { + "50plus": 9619.0, + "lt50": 8554.0, + "total": 9421.0 + }, + "44": { + "50plus": 9369.0, + "lt50": 9465.0, + "total": 9388.0 + }, + "45": { + "50plus": 8479.0, + "lt50": 9020.0, + "total": 8552.0 + }, + "46": { + "50plus": 9225.0, + "lt50": 7990.0, + "total": 8985.0 + }, + "47": { + "50plus": 8712.0, + "lt50": 8365.0, + "total": 8667.0 + }, + "48": { + "50plus": 8514.0, + "lt50": 8737.0, + "total": 8543.0 + }, + "49": { + "50plus": 8533.0, + "lt50": 7702.0, + "total": 8448.0 + }, + "50": { + "50plus": 9452.0, + "lt50": 11460.0, + "total": 10004.0 + }, + "51": { + "50plus": 9157.0, + "lt50": 8141.0, + "total": 8970.0 + }, + "53": { + "50plus": 8592.0, + "lt50": 9146.0, + "total": 8712.0 + }, + "54": { + "50plus": 8897.0, + "lt50": 8763.0, + "total": 8878.0 + }, + "55": { + "50plus": 9618.0, + "lt50": 9260.0, + "total": 9557.0 + }, + "56": { + "50plus": 9155.0, + "lt50": 8691.0, + "total": 8997.0 + }, + "US": { + "50plus": 9024.0, + "lt50": 9034.0, + "total": 9025.0 + } + }, + "text_line": 8651, + "title": "Table II.C.1 Average total single premium (in dollars) per enrolled employee at private-sector establishments that offer health" + } + } + }, + "public_division_2024": { + "employee_plus_one": { + "employee_contribution": { + "rows": { + "East North Central": { + "all_state_and_local": 3617.0, + "state": 3387.0 + }, + "East South Central": { + "all_state_and_local": 3074.0, + "state": 2391.0 + }, + "Middle Atlantic": { + "all_state_and_local": 3294.0, + "state": 3453.0 + }, + "Mountain": { + "all_state_and_local": 3106.0, + "state": 1849.0 + }, + "New England": { + "all_state_and_local": 4414.0, + "state": 3936.0 + }, + "Pacific": { + "all_state_and_local": 2213.0, + "state": 2882.0 + }, + "South Atlantic": { + "all_state_and_local": 4094.0, + "state": 3603.0 + }, + "US": { + "all_state_and_local": 3426.0, + "state": 2973.0 + }, + "West North Central": { + "all_state_and_local": 3343.0, + "state": 1901.0 + }, + "West South Central": { + "all_state_and_local": 4215.0, + "state": 3218.0 + } + }, + "text_line": 3490, + "title": "Table III.E.2 Average total employee contribution (in dollars) per enrolled employee for employee-plus-one coverage through State and" + }, + "premium": { + "rows": { + "East North Central": { + "all_state_and_local": 19183.0, + "state": 18187.0 + }, + "East South Central": { + "all_state_and_local": 14689.0, + "state": 13091.0 + }, + "Middle Atlantic": { + "all_state_and_local": 22282.0, + "state": 19103.0 + }, + "Mountain": { + "all_state_and_local": 16821.0, + "state": 15235.0 + }, + "New England": { + "all_state_and_local": 24306.0, + "state": 23854.0 + }, + "Pacific": { + "all_state_and_local": 19694.0, + "state": 20386.0 + }, + "South Atlantic": { + "all_state_and_local": 17468.0, + "state": 17176.0 + }, + "US": { + "all_state_and_local": 17939.0, + "state": 17352.0 + }, + "West North Central": { + "all_state_and_local": 15648.0, + "state": 14657.0 + }, + "West South Central": { + "all_state_and_local": 14600.0, + "state": 14585.0 + } + }, + "text_line": 3322, + "title": "Table III.E.1 Average total employee-plus-one premium (in dollars) per employee enrolled through State and local government jobs by" + } + }, + "family": { + "employee_contribution": { + "rows": { + "East North Central": { + "all_state_and_local": 4408.0, + "state": 4357.0 + }, + "East South Central": { + "all_state_and_local": 5192.0, + "state": 4598.0 + }, + "Middle Atlantic": { + "all_state_and_local": 4104.0, + "state": 5230.0 + }, + "Mountain": { + "all_state_and_local": 4587.0, + "state": 3518.0 + }, + "New England": { + "all_state_and_local": 7396.0, + "state": 6329.0 + }, + "Pacific": { + "all_state_and_local": 2948.0, + "state": 3226.0 + }, + "South Atlantic": { + "all_state_and_local": 5559.0, + "state": 4133.0 + }, + "US": { + "all_state_and_local": 4763.0, + "state": 4407.0 + }, + "West North Central": { + "all_state_and_local": 4909.0, + "state": 3565.0 + }, + "West South Central": { + "all_state_and_local": 8335.0, + "state": 7268.0 + } + }, + "text_line": 2740, + "title": "Table III.D.2 Average total employee contribution (in dollars) per employee enrolled for family coverage through State and local" + }, + "premium": { + "rows": { + "East North Central": { + "all_state_and_local": 26002.0, + "state": 26404.0 + }, + "East South Central": { + "all_state_and_local": 21068.0, + "state": 19138.0 + }, + "Middle Atlantic": { + "all_state_and_local": 29797.0, + "state": 27676.0 + }, + "Mountain": { + "all_state_and_local": 24041.0, + "state": 22784.0 + }, + "New England": { + "all_state_and_local": 29954.0, + "state": 28114.0 + }, + "Pacific": { + "all_state_and_local": 25909.0, + "state": 27689.0 + }, + "South Atlantic": { + "all_state_and_local": 24204.0, + "state": 23159.0 + }, + "US": { + "all_state_and_local": 26010.0, + "state": 25674.0 + }, + "West North Central": { + "all_state_and_local": 25349.0, + "state": 25951.0 + }, + "West South Central": { + "all_state_and_local": 22682.0, + "state": 22947.0 + } + }, + "text_line": 2572, + "title": "Table III.D.1 Average total family premium (in dollars) per employee enrolled through State and local government jobs by government" + } + }, + "single": { + "employee_contribution": { + "rows": { + "East North Central": { + "all_state_and_local": 1236.0, + "state": 1346.0 + }, + "East South Central": { + "all_state_and_local": 1004.0, + "state": 825.0 + }, + "Middle Atlantic": { + "all_state_and_local": 1314.0, + "state": 1184.0 + }, + "Mountain": { + "all_state_and_local": 781.0, + "state": 935.0 + }, + "New England": { + "all_state_and_local": 2563.0, + "state": 2076.0 + }, + "Pacific": { + "all_state_and_local": 976.0, + "state": 981.0 + }, + "South Atlantic": { + "all_state_and_local": 939.0, + "state": 911.0 + }, + "US": { + "all_state_and_local": 1057.0, + "state": 1024.0 + }, + "West North Central": { + "all_state_and_local": 614.0, + "state": 695.0 + }, + "West South Central": { + "all_state_and_local": 1023.0, + "state": 796.0 + } + }, + "text_line": 1996, + "title": "Table III.C.2 Average total employee contribution (in dollars) per employee enrolled for single coverage through State and local" + }, + "premium": { + "rows": { + "East North Central": { + "all_state_and_local": 9975.0, + "state": 9324.0 + }, + "East South Central": { + "all_state_and_local": 8307.0, + "state": 7555.0 + }, + "Middle Atlantic": { + "all_state_and_local": 11049.0, + "state": 9680.0 + }, + "Mountain": { + "all_state_and_local": 9066.0, + "state": 9452.0 + }, + "New England": { + "all_state_and_local": 11614.0, + "state": 10757.0 + }, + "Pacific": { + "all_state_and_local": 10997.0, + "state": 10633.0 + }, + "South Atlantic": { + "all_state_and_local": 9672.0, + "state": 8891.0 + }, + "US": { + "all_state_and_local": 9687.0, + "state": 9342.0 + }, + "West North Central": { + "all_state_and_local": 8757.0, + "state": 9060.0 + }, + "West South Central": { + "all_state_and_local": 8302.0, + "state": 8564.0 + } + }, + "text_line": 1828, + "title": "Table III.C.1 Average total single premium (in dollars) per employee enrolled through State and local government jobs by government" + } + } + }, + "schema_version": 2, + "sources": { + "private_state_2024": { + "description": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025 and carry the 2024 private-sector enrollment", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", + "size_bytes": 10687163, + "survey_year": 2024, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf" + }, + "private_state_2025": { + "description": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee), II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent of enrollees whose coverage required no employee contribution), and the enrollment rows II.B.1, II.B.2, II.B.2.b and II.C/D/E.4", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", + "size_bytes": 12530005, + "survey_year": 2025, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf" + }, + "public_division_2024": { + "description": "MEPS-IC 2024 Series III (State and local government, by government type and size and census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released).", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79", + "size_bytes": 1765717, + "survey_year": 2024, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_3/2024/ic24_iiia_g.pdf" + } + }, + "state_census_division": { + "01": "East South Central", + "02": "Pacific", + "04": "Mountain", + "05": "West South Central", + "06": "Pacific", + "08": "Mountain", + "09": "New England", + "10": "South Atlantic", + "11": "South Atlantic", + "12": "South Atlantic", + "13": "South Atlantic", + "15": "Pacific", + "16": "Mountain", + "17": "East North Central", + "18": "East North Central", + "19": "West North Central", + "20": "West North Central", + "21": "East South Central", + "22": "West South Central", + "23": "New England", + "24": "South Atlantic", + "25": "New England", + "26": "East North Central", + "27": "West North Central", + "28": "East South Central", + "29": "West North Central", + "30": "Mountain", + "31": "West North Central", + "32": "Mountain", + "33": "New England", + "34": "Middle Atlantic", + "35": "Mountain", + "36": "Middle Atlantic", + "37": "South Atlantic", + "38": "West North Central", + "39": "East North Central", + "40": "West South Central", + "41": "Pacific", + "42": "Middle Atlantic", + "44": "New England", + "45": "South Atlantic", + "46": "West North Central", + "47": "East South Central", + "48": "West South Central", + "49": "Mountain", + "50": "New England", + "51": "South Atlantic", + "53": "Pacific", + "54": "South Atlantic", + "55": "East North Central", + "56": "Mountain" + }, + "tiers": [ + "single", + "employee_plus_one", + "family" + ] +} diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py new file mode 100644 index 0000000000..86f612526c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py @@ -0,0 +1,1257 @@ +"""MEPS-IC employer premiums for ESI policyholders. + +Microcosm #454. PolicyEngine-US 2.2.1 reads a person input that no Microcosm +stage produced: ``employer_sponsored_insurance_premiums``, "annual +employer-paid health insurance premiums", one of +``gov.household.cbo_market_income_additions`` (CBO household market income). + +The retired eCPS derivation (archived 42ed5d45 ``datasets/cps/cps.py`` +L197-271) read the CPS ASEC current employment-based coverage fields, which the +pinned H5 inputs omit; :mod:`.asec_census_person_columns` now restores them, +with employment status and employer size, from the pinned Census person files. + +Employer premium +---------------- +Both national accounts that publish an employer-contribution total (CMS NHE +Table 24 and BEA NIPA Table 7.8 line 17) count contributions for active +**and** retired employees, and both build the non-federal part from MEPS-IC. +So the stage prices every **current ESI policyholder** (``NOW_OWNGRP`` 1, +employed or not: the accounts' universe), scales that universe to the account +total, and writes the result only for the **employed** policyholders +(``PEMLR`` 1-2) whose current job has an employer (``PEIO1COW`` 1-6). The +column is the compensation of a current job: dependents carry nothing (the +policyholder carries the premium), and retired, unemployed and other +non-employed policyholders, self-employed-unincorporated workers and workers +without pay carry nothing either. Their scaled share stays out of the column +rather than being loaded onto workers. + +Each policyholder's raw employer share comes from the MEPS-IC cell of their +coverage tier (``NOW_GRPFTYP2``: family, self plus one, self-only), employer +sector (``PEIO1COW``) and, for private employers, State and firm size +(``NOEMP`` under 50 / 50 or more): + +* employer paid all of the premium (``NOW_HIPAID`` 1): the cell's average + total premium; +* employer paid some (2): the average total premium minus the contribution of + an employee who pays one. MEPS-IC averages the employee contribution over + every enrollee, including those who pay nothing, so the stage divides it by + one minus the published share of enrollees whose coverage required no + contribution (Tables II.C/D/E.4.a, national by firm size; most State cells + of those tables are flagged unreliable). A cohort with MEPS-IC's own + all/some mix then reproduces the cell's published employer mean; +* employer paid none (3): zero. MEPS-IC publishes no share of enrollees who + pay the whole premium, so they are treated as outside its averages. + +Private-sector cells are MEPS-IC 2025 Series II (State by firm size). +Government cells are MEPS-IC 2024 Series III by census division (the latest +year published), aged to 2025 by the private-sector national 2025/2024 ratio of +the same tier and measure: State government employees take the +State-government column, local and federal employees the all-governments +column (MEPS-IC does not survey the federal government, so that is a +stand-in). Series III publishes no no-contribution share, so government cells +take the private 50-or-more share (a stand-in). A policyholder with no +employer class (not employed and never asked, self-employed unincorporated or +without pay) takes the State's all-sizes private cell. No MEPS-IC table prices +retiree coverage, so non-employed policyholders are priced as active +employees; if retiree plans cost less, the employed column is understated. +AHRQ suppresses nine small-firm employee-contribution cells; each takes the +national small-firm cell times the State's total-to-national ratio. + +One factor scales every policyholder's raw share so the weighted total equals +BEA NIPA Table 7.8 line 17 (series B4923C, "Private group health insurance") +for the build year, the anchor decision d1317 chose. BEA's State Personal +Income methodology (December 2025, paragraphs 3.18-3.19) says its MEPS source +"covers both health insurance purchased by employers for their active and +retired employees" and its federal estimate covers "active and retired federal +civilian employees". The pinned US Chronicle feed carries the same values +(record sets ``bea_nipa.cy.private_group_health_insurance``), and the +release tool refuses a feed whose values differ from the stage's. + +CMS NHE Table 24's employer contribution to private health insurance premiums +is a second estimate of the same concept, 7.2% higher for 2024 (its +methodology paper: "active employees, continuation of health coverage +(COBRA), and retirees"). It is recorded beside the anchor verdict, as is +MEPS-IC's own private-sector active-employee total, which the private part of +the column matches within half a percent on the pinned pools. Neither is +gated. + +Not produced here +----------------- +``pre_tax_health_insurance_premiums``, the employee share paid by pre-tax +payroll deduction, has no producer yet. PolicyEngine-US treats it as disjoint +from the other premium inputs (PolicyEngine/policyengine-us#10046), so +producing it means moving those dollars out of the reported premium the base +carries, on an engine that counts the pre-tax input where the reported premium +was counted. That is a separate change. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from importlib.resources import files +from types import MappingProxyType +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.gates import GateResult +from microcosm.build.source_manifest import ( + SourceOperationSpec, + SourceStageSpec, + load_source_manifest, +) +from microcosm.build.source_runtime import ( + SourceRuntimeConfig, + SourceRuntimeContext, + SourceRuntimeError, + run_source_stage, +) +from microcosm.frame import Frame +from microcosm.frame.units import US_SCHEMA + +__all__ = [ + "US_ESI_EMPLOYER_PREMIUM_COLUMN", + "US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS", + "US_ESI_PREMIUMS_OUTPUT_COLUMNS", + "US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS", + "US_ESI_PREMIUMS_STAGE_NAME", + "derive_us_employer_esi_premiums_from_manifest", + "load_meps_ic_esi_premium_cells", + "meps_ic_private_active_employer_totals", + "refuse_unassigned_us_esi_premiums", + "us_esi_premiums_anchor_feed_mismatches", + "us_esi_premiums_anchor_gate", + "us_esi_premiums_signal_gate", + "us_esi_premiums_stage_spec", + "us_esi_premiums_summary", + "with_us_esi_premium_inputs", +] + +US_ESI_PREMIUMS_STAGE_NAME = "meps_esi_premiums" +US_ESI_EMPLOYER_PREMIUM_COLUMN = "employer_sponsored_insurance_premiums" +US_ESI_PREMIUMS_OUTPUT_COLUMNS: tuple[str, ...] = (US_ESI_EMPLOYER_PREMIUM_COLUMN,) +US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS = US_ESI_PREMIUMS_OUTPUT_COLUMNS + +#: Raw CPS ASEC person columns the stage reads (the six NOW_*/PEMLR/NOEMP +#: columns through :mod:`.asec_census_person_columns`), plus the household +#: State (``state_fips``, from GESTFIPS), joined onto the person table by +#: ``person_household_id`` when it lives on the household table. +US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS: tuple[str, ...] = ( + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP", + "PEIO1COW", + "state_fips", +) + +#: Census CPS ASEC person codes (survey years 2023-2026 codebooks, matching the +#: pinned person members on 2026-10-10). +_CODE_DOMAINS: Mapping[str, frozenset[int]] = MappingProxyType( + { + "NOW_OWNGRP": frozenset({0, 1, 2}), + "NOW_HIPAID": frozenset({0, 1, 2, 3}), + "NOW_GRPFTYP": frozenset({0, 1, 2}), + "NOW_GRPFTYP2": frozenset({0, 1, 2, 3}), + "PEMLR": frozenset(range(8)), + "NOEMP": frozenset(range(7)), + "PEIO1COW": frozenset(range(9)), + } +) +_POLICYHOLDER = 1 +_EMPLOYED = (1, 2) +_PAYS_ALL, _PAYS_SOME, _PAYS_NONE = 1, 2, 3 +#: NOW_GRPFTYP2 -> MEPS-IC tier; NOW_GRPFTYP must agree (1 family, 2 self-only). +_TIER_BY_CODE: Mapping[int, str] = MappingProxyType( + {1: "family", 2: "employee_plus_one", 3: "single"} +) +_GRPFTYP_FOR_GRPFTYP2: Mapping[int, int] = MappingProxyType({1: 1, 2: 1, 3: 2}) +#: NOEMP ("Work experience, persons who work for employer"): 1 under 10 and +#: 2 10-49 -> under 50; 3-6 -> 50 or more; 0 (not in universe) -> the State's +#: all-sizes cell. +_SIZE_BY_NOEMP: Mapping[int, str] = MappingProxyType( + { + 0: "total", + 1: "lt50", + 2: "lt50", + 3: "50plus", + 4: "50plus", + 5: "50plus", + 6: "50plus", + } +) +#: PEIO1COW: 1 federal, 2 state, 3 local, 4 private for profit, 5 private +#: nonprofit, 6 self-employed incorporated, 7 self-employed unincorporated, +#: 8 without pay. +_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT = 1, 2, 3 +_GOVERNMENT_EMPLOYER_CODES = (_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT) +_PRIVATE_EMPLOYER_CODES = (4, 5, 6) +_EMPLOYER_SECTOR_CODES = (_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT, 4, 5, 6) + +_CELLS_RESOURCE = "meps_ic_esi_premium_cells.json" +_CELLS_SHA256 = "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93" +_PERSON_WEIGHT_COLUMN = "person_weight" + +_BEA_SECTION7_URL = ( + "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx" +) +_NHE_TABLES_URL = "https://www.cms.gov/files/zip/nhe-tables.zip" + +#: BEA NIPA Table 7.8 line 17 (B4923C), millions of dollars converted to +#: dollars, from the annual update published 2026-09-30 (file created +#: 2026-09-28; that update revised 2024 from the $1,002.9B the retired us-data +#: loss matrix pinned). The anchor decision d1317 chose. The pinned US +#: Chronicle feed banks the same three values (chronicle#331). +EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( + { + "source": _BEA_SECTION7_URL, + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "row": "line 17, series B4923C, Private group health insurance", + "line": 17, + "series": "B4923C", + "published": "2026-09-30", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", + "coverage": ( + "employer contributions for active and retired employees (BEA " + "State Personal Income methodology, December 2025, paragraphs " + "3.18-3.19)" + ), + # Integer dollars: the manifest round-trips through YAML. + "values": { + "2023": 923_195_000_000, + "2024": 977_034_000_000, + "2025": 1_027_929_000_000, + }, + } +) +#: CMS NHE Table 24 (2024 National Health Expenditure Accounts, data through +#: CY2024), row "Employer Contribution to Private Health Insurance Premiums": +#: private, federal and State and local employers, for active employees, COBRA +#: enrollees and retirees; banked in the pinned feed as +#: cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums. A second estimate +#: of the same concept; recorded, not gated. ``sha256`` is the digest of the +#: zip archive at ``source`` (520,391 bytes), not of the Table 24 workbook +#: inside it. +EMPLOYER_PREMIUM_CROSS_CHECK: Mapping[str, Any] = MappingProxyType( + { + "source": _NHE_TABLES_URL, + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "row": "Employer Contribution to Private Health Insurance Premiums", + "values": {"2023": 975_700_000_000, "2024": 1_047_000_000_000}, + "relation": ( + "second estimate of the same concept: employer contributions for " + "active employees, COBRA enrollees and retirees (NHEA methodology " + "paper, 2024, p. 29)" + ), + } +) +#: Tolerance of the calibrated release aggregate around the anchor, over the +#: anchor's universe (every policyholder). The stage pins the pre-calibration +#: aggregate exactly, so the band is the room a release gives calibration and +#: sparse selection to move it. It is a chosen release criterion, not a +#: measured error: no release has calibrated this column yet. +ANCHOR_RELATIVE_TOLERANCE = 0.05 +#: Weighted-raw employer share per positive employed holder, before scaling. +#: MEPS-IC 2025 employer shares run $7.2k (single) to $19.0k (family) and the +#: pinned pools measure $12.0-12.1k. The band catches a gross unit error in the +#: cell table (monthly for annual, cents for dollars), which the scale factor +#: would otherwise hide; it does not catch a level error inside the band. +_RAW_MEAN_BAND = (6_000.0, 20_000.0) +#: Weighted share of all persons with a positive employer premium. Measured +#: on both pinned pools: 22.7%. +_POSITIVE_SHARE_BAND = (0.15, 0.32) +#: Employed policyholders' share of the anchor-universe employer total. +#: Measured on the pinned pools: 88.4% and 88.2%. A release whose calibration +#: moves the split far from that has reweighted workers against retirees. +_EMPLOYED_SHARE_BAND = (0.80, 0.95) + +_EMPLOYER_PREMIUM_PARAMETERS: Mapping[str, Any] = MappingProxyType( + { + "cells_resource": f"microcosm.build.us_runtime.data/{_CELLS_RESOURCE}", + "cells_sha256": _CELLS_SHA256, + "anchor_universe": "NOW_OWNGRP == 1", + "universe": ( + "NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6)" + ), + "tier_column": "NOW_GRPFTYP2", + "size_column": "NOEMP", + "sector_column": "PEIO1COW", + "payment_column": "NOW_HIPAID", + "payment_rule": ( + "1 (employer paid all): average total premium; 2 (some): average " + "total premium minus average employee contribution / (1 - " + "no-contribution share); 3 (none): 0" + ), + "no_contribution_share": ( + "MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; " + "government cells take the 50-or-more share" + ), + "other_policyholder_cell": ( + "PEIO1COW outside 1-6: the State's all-sizes private cell" + ), + "suppressed_cell_fallback": ( + "national firm-size cell times the State total-to-national ratio" + ), + "government_cell_aging": ( + "MEPS-IC 2024 Series III x private-sector national 2025/2024 " + "ratio of the same tier and measure" + ), + "scaling": ( + "one factor, anchor / weighted raw share of the anchor universe; " + "written to the universe only" + ), + "anchor": { + "source": EMPLOYER_PREMIUM_ANCHOR["source"], + "sha256": EMPLOYER_PREMIUM_ANCHOR["sha256"], + "table": EMPLOYER_PREMIUM_ANCHOR["table"], + "row": EMPLOYER_PREMIUM_ANCHOR["row"], + "values": dict(EMPLOYER_PREMIUM_ANCHOR["values"]), + }, + } +) + + +def us_esi_premiums_stage_spec() -> SourceStageSpec: + """Load the ``meps_esi_premiums`` stage and pin it to this contract.""" + + manifest = load_source_manifest( + files("microcosm.build.us").joinpath("source_stages.json") + ) + spec = manifest.stage_map().get(US_ESI_PREMIUMS_STAGE_NAME) + if spec is None: + raise ValueError( + f"US source manifest declares no {US_ESI_PREMIUMS_STAGE_NAME!r} stage." + ) + if spec.grain != "person": + raise ValueError("US ESI premium stage must have person grain.") + if tuple(spec.outputs) != US_ESI_PREMIUMS_OUTPUT_COLUMNS: + raise ValueError("US ESI premium manifest outputs drifted from the runtime.") + expected = ( + ("read_table", {"table": "person", "weight": _PERSON_WEIGHT_COLUMN}), + ( + "derive_employer_sponsored_insurance_premiums", + _plain(_EMPLOYER_PREMIUM_PARAMETERS), + ), + ) + actual = tuple( + (operation.kind, dict(operation.parameters)) for operation in spec.operations + ) + if actual != expected: + raise ValueError( + "US ESI premium operations drifted from the reviewed MEPS-IC cell " + "assignment contract." + ) + return spec + + +def _plain(value: Any) -> Any: + if isinstance(value, Mapping): + return {key: _plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(item) for item in value] + return value + + +def load_meps_ic_esi_premium_cells() -> Mapping[str, Any]: + """The packaged MEPS-IC cell table, verified against its pinned digest.""" + + raw = ( + files("microcosm.build.us_runtime.data").joinpath(_CELLS_RESOURCE).read_bytes() + ) + digest = hashlib.sha256(raw).hexdigest() + if digest != _CELLS_SHA256: + raise SourceRuntimeError( + f"{_CELLS_RESOURCE} digest {digest} is not the reviewed {_CELLS_SHA256}; " + "regenerate it with tools/build_us_meps_ic_esi_cells.py and re-pin." + ) + return json.loads(raw) + + +@dataclass(frozen=True) +class _EsiCodes: + owner: np.ndarray + hipaid: np.ndarray + tier: np.ndarray + employed: np.ndarray + noemp: np.ndarray + sector: np.ndarray + state_fips: np.ndarray + + @property + def policyholder(self) -> np.ndarray: + """Every current ESI policyholder: the universe the anchor measures.""" + + return self.owner == _POLICYHOLDER + + @property + def universe(self) -> np.ndarray: + """Employed policyholders with an employer: the column's universe.""" + + return ( + self.policyholder + & self.employed + & np.isin(self.sector, _EMPLOYER_SECTOR_CODES) + ) + + +def _integer_codes(person: pd.DataFrame, column: str) -> np.ndarray: + values = pd.to_numeric(person[column], errors="coerce") + numeric = values.to_numpy(dtype=np.float64, na_value=np.nan) + bad = ~np.isfinite(numeric) | (numeric != np.round(numeric)) + if bad.any(): + raise SourceRuntimeError( + f"US ESI premium stage: {column} has {int(bad.sum())} missing or " + "non-integer row(s); the Census column was not restored for every " + "vintage (asec_census_person_columns) and is never defaulted." + ) + codes = numeric.astype(np.int64) + domain = _CODE_DOMAINS.get(column) + if domain is not None: + outside = ~np.isin(codes, sorted(domain)) + if outside.any(): + raise SourceRuntimeError( + f"US ESI premium stage: {column} has {int(outside.sum())} code(s) " + f"outside the Census codebook {sorted(domain)}, e.g. " + f"{sorted(set(codes[outside].tolist()))[:5]}." + ) + return codes + + +def _esi_person_codes(person: pd.DataFrame) -> _EsiCodes: + """Validate the raw ASEC coverage, employment and employer fields. + + Reads NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR, NOEMP and + PEIO1COW, and refuses the Census universe relations the pinned person + members satisfy exactly: a policyholder reports a payment status and a + tier, everyone else reports neither, and NOW_GRPFTYP (family/self-only) + agrees with NOW_GRPFTYP2. + """ + + missing = [ + column + for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column not in person.columns + ] + if missing: + raise SourceRuntimeError( + "US ESI premium stage is source-unavailable: the person table lacks " + f"{missing}. Restore them from the pinned Census person files " + "(asec_census_person_columns); they are never proxied or defaulted." + ) + owner = _integer_codes(person, "NOW_OWNGRP") + hipaid = _integer_codes(person, "NOW_HIPAID") + grpftyp = _integer_codes(person, "NOW_GRPFTYP") + tier = _integer_codes(person, "NOW_GRPFTYP2") + pemlr = _integer_codes(person, "PEMLR") + noemp = _integer_codes(person, "NOEMP") + sector = _integer_codes(person, "PEIO1COW") + state_fips = _integer_codes(person, "state_fips") + holder = owner == _POLICYHOLDER + violations = { + "policyholder_without_payment_status": int( + (holder & ~np.isin(hipaid, [_PAYS_ALL, _PAYS_SOME, _PAYS_NONE])).sum() + ), + "policyholder_without_tier": int((holder & ~np.isin(tier, [1, 2, 3])).sum()), + "non_policyholder_with_payment_status": int((~holder & (hipaid != 0)).sum()), + "non_policyholder_with_tier": int((~holder & (tier != 0)).sum()), + "grpftyp_disagrees_with_grpftyp2": int( + ( + holder + & ( + grpftyp + != pd.Series(tier) + .map(dict(_GRPFTYP_FOR_GRPFTYP2)) + .fillna(-1) + .to_numpy() + ) + ).sum() + ), + } + if any(violations.values()): + rendered = ", ".join( + f"{key}={value}" for key, value in violations.items() if value + ) + raise SourceRuntimeError( + f"US ESI premium stage refused the ASEC coverage fields: {rendered}." + ) + return _EsiCodes( + owner=owner, + hipaid=hipaid, + tier=tier, + employed=np.isin(pemlr, _EMPLOYED), + noemp=noemp, + sector=sector, + state_fips=state_fips, + ) + + +def _private_cell( + cells: Mapping[str, Any], tier: str, measure: str, state: str, size: str +) -> float: + rows = cells["private_state_2025"][tier][measure]["rows"] + if state not in rows: + raise SourceRuntimeError( + f"US ESI premium stage: no MEPS-IC row for State {state}." + ) + value = rows[state][size] + if value is None: + # AHRQ suppressed the cell: national firm-size cell scaled by the + # State's all-sizes level relative to the nation. + national = rows["US"] + value = national[size] * rows[state]["total"] / national["total"] + return float(value) + + +def _government_cell( + cells: Mapping[str, Any], tier: str, measure: str, division: str, column: str +) -> float: + value = cells["public_division_2024"][tier][measure]["rows"][division][column] + current = cells["private_state_2025"][tier][measure]["rows"]["US"]["total"] + prior = cells["private_national_2024"][tier][measure]["total"] + return float(value) * float(current) / float(prior) + + +def _no_contribution_share(cells: Mapping[str, Any], tier: str, size: str) -> float: + return float(cells["no_contribution_share_2025"][tier][size]) / 100.0 + + +def _cell_values( + codes: _EsiCodes, cells: Mapping[str, Any] +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Each policyholder's cell: premium, employee contribution, no-contribution share. + + NaN for everyone who is not a policyholder. + """ + + premium = np.full(len(codes.owner), np.nan) + contribution = np.full(len(codes.owner), np.nan) + no_contribution = np.full(len(codes.owner), np.nan) + divisions = cells["state_census_division"] + keys = pd.DataFrame( + { + "tier": codes.tier, + "noemp": codes.noemp, + "sector": codes.sector, + "state": codes.state_fips, + } + ) + for (tier_code, noemp, sector, state), index in ( + keys[codes.policyholder] + .groupby(["tier", "noemp", "sector", "state"]) + .groups.items() + ): + tier = _TIER_BY_CODE[int(tier_code)] + state_key = f"{int(state):02d}" + if state_key not in divisions: + raise SourceRuntimeError( + f"US ESI premium stage: State FIPS {int(state)} has no MEPS-IC cell." + ) + if int(sector) in _GOVERNMENT_EMPLOYER_CODES: + column = ( + "state" if int(sector) == _STATE_GOVERNMENT else "all_state_and_local" + ) + values = [ + _government_cell(cells, tier, measure, divisions[state_key], column) + for measure in ("premium", "employee_contribution") + ] + # Series III publishes no no-contribution share. + size = "50plus" + else: + # Private employers by firm size; a policyholder with no employer + # class takes the State's all-sizes private cell. + size = ( + _SIZE_BY_NOEMP[int(noemp)] + if int(sector) in _PRIVATE_EMPLOYER_CODES + else "total" + ) + values = [ + _private_cell(cells, tier, measure, state_key, size) + for measure in ("premium", "employee_contribution") + ] + rows = np.asarray(index, dtype=np.int64) + premium[rows], contribution[rows] = values + no_contribution[rows] = _no_contribution_share(cells, tier, size) + return premium, contribution, no_contribution + + +def _raw_employer_share( + codes: _EsiCodes, + premium: np.ndarray, + contribution: np.ndarray, + no_contribution: np.ndarray, +) -> np.ndarray: + """Unscaled employer share of every policyholder (zero for everyone else).""" + + raw = np.zeros(len(codes.owner), dtype=np.float64) + holder = codes.policyholder + pays_all = holder & (codes.hipaid == _PAYS_ALL) + pays_some = holder & (codes.hipaid == _PAYS_SOME) + raw[pays_all] = premium[pays_all] + # The published contribution averages over enrollees who pay nothing; + # dividing by the share who pay gives the contribution of those who do. + raw[pays_some] = np.clip( + premium[pays_some] + - contribution[pays_some] / (1.0 - no_contribution[pays_some]), + 0.0, + None, + ) + return raw + + +def _person_raw_shares( + person: pd.DataFrame, +) -> tuple[_EsiCodes, np.ndarray, np.ndarray, np.ndarray]: + """Codes, raw employer share, cell premium and cell employee contribution.""" + + codes = _esi_person_codes(person) + premium, contribution, no_contribution = _cell_values( + codes, load_meps_ic_esi_premium_cells() + ) + raw = _raw_employer_share(codes, premium, contribution, no_contribution) + return codes, raw, premium, contribution + + +def meps_ic_private_active_employer_totals( + cells: Mapping[str, Any], +) -> dict[str, float]: + """MEPS-IC's own private-sector employer total for enrolled employees. + + Enrolled employees (employees x the share in establishments that offer + health insurance x the share enrolled there) times each tier's share of + enrollees and its average premium less average employee contribution, + from the United States rows of the pinned Series II tables. It covers + active private-sector employees only, so it checks the private part of + the column against the survey the cells come from. + """ + + national = { + "2024": lambda tier, measure: cells["private_national_2024"][tier][measure][ + "total" + ], + "2025": lambda tier, measure: cells["private_state_2025"][tier][measure][ + "rows" + ]["US"]["total"], + } + totals: dict[str, float] = {} + for year, rows in cells["private_enrollment_national"].items(): + enrolled = ( + float(rows["employees"]["total"]) + * float(rows["offer_percent"]["total"]) + / 100.0 + * float(rows["enrolled_percent"]["total"]) + / 100.0 + ) + totals[year] = sum( + enrolled + * float(rows["tier_share_percent"][tier]["total"]) + / 100.0 + * ( + float(national[year](tier, "premium")) + - float(national[year](tier, "employee_contribution")) + ) + for tier in _TIER_BY_CODE.values() + ) + return totals + + +def us_esi_premiums_anchor_feed_mismatches( + facts: Iterable[Mapping[str, Any]], +) -> list[str]: + """Where the Chronicle feed's B4923C facts differ from the stage's anchor. + + The stage pins the BEA values it scales to; the pinned US feed banks the + same series (``bea_nipa.cy.private_group_health_insurance``). The + release tool calls this on the facts it compiles targets from, so the + producer's anchor and the banked fact cannot drift apart unseen. Returns + one message per anchor year whose fact is missing, repeated or different. + """ + + template = str(EMPLOYER_PREMIUM_ANCHOR["feed_record_set"]) + wanted = { + template.format(year=year): (year, int(value)) + for year, value in EMPLOYER_PREMIUM_ANCHOR["values"].items() + } + found: dict[str, list[Any]] = {record_set: [] for record_set in wanted} + for fact in facts: + layout = fact.get("layout") + record_set = ( + layout.get("record_set_id") if isinstance(layout, Mapping) else None + ) + if record_set in found: + found[record_set].append(fact.get("value")) + problems: list[str] = [] + for record_set, (year, expected) in sorted(wanted.items()): + values = found[record_set] + if not values: + problems.append(f"{record_set}: no fact in the feed ({year}).") + elif len(values) > 1: + problems.append(f"{record_set}: {len(values)} facts in the feed ({year}).") + elif values[0] != expected: + problems.append( + f"{record_set}: feed value {values[0]!r} is not the stage's " + f"anchor {expected} ({year})." + ) + return problems + + +def _anchor(year: int) -> float: + values = EMPLOYER_PREMIUM_ANCHOR["values"] + if str(year) not in values: + raise SourceRuntimeError( + f"US ESI premium stage has no BEA NIPA 7.8 line 17 anchor for {year}; pinned " + f"years: {sorted(values)}." + ) + return float(values[str(year)]) + + +def _check_parameters( + operation: SourceOperationSpec, expected: Mapping[str, Any] +) -> None: + if dict(operation.parameters) != _plain(expected): + raise SourceRuntimeError( + f"US ESI premium operation {operation.kind!r} parameters drifted from " + "the reviewed contract." + ) + + +def _person_weights(frame: pd.DataFrame) -> np.ndarray: + if _PERSON_WEIGHT_COLUMN not in frame.columns: + raise SourceRuntimeError("US ESI premium stage requires person weights.") + weights = frame[_PERSON_WEIGHT_COLUMN].to_numpy(dtype=np.float64) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise SourceRuntimeError( + "US ESI premium stage requires finite nonnegative weights." + ) + return weights + + +def derive_us_employer_esi_premiums_from_manifest( + frame: pd.DataFrame | None, + operation: SourceOperationSpec, + context: SourceRuntimeContext, +) -> pd.DataFrame: + """Assign MEPS-IC employer shares and scale them to the BEA anchor. + + The factor is set over every policyholder, the anchor's universe; the + column carries the employed policyholders' part. + """ + + if operation.kind != "derive_employer_sponsored_insurance_premiums": + raise SourceRuntimeError( + f"Unexpected ESI premium operation {operation.kind!r}." + ) + if frame is None: + raise SourceRuntimeError("ESI premium assignment requires the person table.") + _check_parameters(operation, _EMPLOYER_PREMIUM_PARAMETERS) + codes, raw, _premium, _contribution = _person_raw_shares(frame) + weights = _person_weights(frame) + universe = codes.universe + if not float(weights[universe] @ raw[universe]) > 0: + raise SourceRuntimeError( + "US ESI premium stage found no weighted employer-paid premium mass to " + "scale: no employed policyholder with an employer share." + ) + scale = _anchor(context.config.target_year) / float(weights @ raw) + result = frame.copy(deep=True) + result[US_ESI_EMPLOYER_PREMIUM_COLUMN] = np.where(universe, raw * scale, 0.0) + return result + + +def _stable_person_keys(frame: pd.DataFrame) -> pd.Series: + if {"source_year", "source_household_id", "source_person_id"} <= set(frame.columns): + return ( + frame["source_year"].astype(str) + + ":" + + frame["source_household_id"].astype(str) + + ":" + + frame["source_person_id"].astype(str) + ) + return frame["person_id"].astype(str) + + +def _person_with_state(frame: Frame) -> pd.DataFrame: + """The person table with the household ``state_fips`` joined on.""" + + person = frame.table("person").copy(deep=True) + if "state_fips" in person.columns: + return person + household = frame.table("household") + if "state_fips" not in household.columns: + return person + state = household.set_index("household_id")["state_fips"] + if not state.index.is_unique: + raise SourceRuntimeError("US ESI premium stage: household ids repeat.") + joined = person["person_household_id"].map(state) + if joined.isna().any(): + raise SourceRuntimeError( + f"US ESI premium stage: {int(joined.isna().sum())} person(s) have no " + "household state_fips." + ) + person["state_fips"] = joined.to_numpy() + return person + + +def _outputs_carry_signal(person: pd.DataFrame) -> bool: + return all( + column in person.columns and person[column].dropna().nunique() > 1 + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + ) + + +def with_us_esi_premium_inputs(frame: Frame, *, seed: int, time_period: int) -> Frame: + """Run the ``meps_esi_premiums`` stage over a US frame. + + A frame that already carries the output with signal and passes the signal + gate is returned unchanged (idempotent). Otherwise the person table must + still carry the raw ASEC columns; the stage never defaults them. The + assignment is deterministic: ``seed`` only fills the runtime config that + every source stage takes. + """ + + if frame.schema != US_SCHEMA: + raise ValueError("US ESI premium inputs require the US schema.") + person = frame.table("person") + if _outputs_carry_signal(person): + gate = us_esi_premiums_signal_gate(frame) + if not gate.passed: + raise SourceRuntimeError( + "US ESI premium preexisting inputs fail the signal gate: " + + "; ".join(gate.failures) + ) + return frame + stage_person = _person_with_state(frame) + stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + output = run_source_stage( + us_esi_premiums_stage_spec(), + tables={"person": stage_person}, + operation_handlers={ + "derive_employer_sponsored_insurance_premiums": ( + derive_us_employer_esi_premiums_from_manifest + ), + }, + config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + ) + aligned = output.set_index("person_id").reindex(person["person_id"]) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + values = aligned[column].to_numpy(dtype=np.float64) + if not np.isfinite(values).all(): + raise ValueError(f"US ESI premium stage output does not cover {column!r}.") + tables["person"][column] = values + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def refuse_unassigned_us_esi_premiums(frame: Frame, *, consumer: str) -> None: + """Refuse a frame whose ESI employer premium input is null on some rows. + + A lineage that pools rows the stage never ran on (ACS-spine rows beside an + ASEC donor that carries the column) leaves it null there. Filling those + with the engine default would ship a column that is zero on part of the + population yet looks populated, and no anchor gate runs on that lineage, so + the consumer refuses instead. + """ + + person = frame.table("person") + gaps = { + column: int(person[column].isna().sum()) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + if column in person.columns and person[column].isna().any() + } + if gaps: + raise SourceRuntimeError( + f"{consumer}: ESI premium input(s) are null on some rows {gaps}. The " + "meps_esi_premiums stage (microcosm#454) ran on the ASEC rows only; " + "its output is never filled with an engine default. Assign or " + "transfer it for every spine before building this lineage." + ) + + +def _weighted( + weights: np.ndarray, values: np.ndarray, mask: np.ndarray | None = None +) -> float: + if mask is None: + return float(weights @ values) + return float(weights[mask] @ values[mask]) + + +def _clone_disagreements(person: pd.DataFrame) -> int: + if not {"source_year", "source_household_id", "source_person_id"} <= set( + person.columns + ): + return 0 + keys = _stable_person_keys(person) + work = person[list(US_ESI_PREMIUMS_OUTPUT_COLUMNS)].copy() + work["_key"] = keys.to_numpy() + return int( + work.groupby("_key", sort=False)[list(US_ESI_PREMIUMS_OUTPUT_COLUMNS)] + .nunique() + .gt(1) + .any(axis=1) + .sum() + ) + + +def us_esi_premiums_summary(frame: Frame) -> dict[str, object]: + """Weighted ESI premium lineage and structure, for gates and manifests. + + It recomputes the raw MEPS-IC shares from the raw ASEC columns, so the + scale factor, the anchor-universe total and every structural zero are + checked against the cells rather than trusted. A frame that lacks those + columns gets the column totals only, with ``source_columns_missing`` + naming what is absent; both gates fail on it. + """ + + person = frame.table("person") + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + positive = employer > 0 + total_weight = float(weights.sum()) + holders = float(weights[positive].sum()) + person = _person_with_state(frame) + summary: dict[str, object] = { + "employer_premium_total": _weighted(weights, employer), + "employer_premium_positive_persons": holders, + "employer_premium_mean_per_positive_person": ( + _weighted(weights, employer, positive) / holders if holders else 0.0 + ), + "employer_premium_positive_share": holders / total_weight + if total_weight + else 0.0, + "nonfinite_rows": int((~np.isfinite(employer)).sum()), + "negative_rows": int((employer < 0).sum()), + "clone_disagreement_source_persons": _clone_disagreements(person), + "anchor": dict(EMPLOYER_PREMIUM_ANCHOR), + "cross_check": dict(EMPLOYER_PREMIUM_CROSS_CHECK), + "cells_sha256": _CELLS_SHA256, + "source_columns_missing": [ + column + for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column not in person.columns + ], + } + if summary["source_columns_missing"]: + return summary + codes, raw, premium, contribution = _person_raw_shares(person) + universe = codes.universe + holder = codes.policyholder + other = holder & ~universe + raw_positive = universe & (raw > 0) + raw_universe_total = _weighted(weights, raw, universe) + raw_anchor_total = _weighted(weights, raw) + scale = ( + summary["employer_premium_total"] / raw_universe_total + if raw_universe_total + else float("nan") + ) + by_tier = { + tier: _weighted(weights, employer, universe & (codes.tier == code)) + for code, tier in _TIER_BY_CODE.items() + } + by_sector = { + "private": _weighted( + weights, employer, np.isin(codes.sector, _PRIVATE_EMPLOYER_CODES) + ), + "federal": _weighted(weights, employer, codes.sector == _FEDERAL), + "state_and_local": _weighted( + weights, + employer, + np.isin(codes.sector, [_STATE_GOVERNMENT, _LOCAL_GOVERNMENT]), + ), + } + with np.errstate(invalid="ignore", divide="ignore"): + ratio = np.where( + raw_positive, employer / np.where(raw_positive, raw, 1.0), np.nan + ) + # MEPS-IC's published employer mean of a cell is premium less average + # contribution over every enrollee the employer pays for. + employer_pays = universe & np.isin(codes.hipaid, [_PAYS_ALL, _PAYS_SOME]) + published = _weighted(weights, premium - contribution, employer_pays) + summary |= { + "weighted_policyholders": float(weights[holder].sum()), + "weighted_employed_policyholders": float(weights[universe].sum()), + "weighted_employed_policyholders_by_payment": { + name: float(weights[universe & (codes.hipaid == code)].sum()) + for name, code in ( + ("all", _PAYS_ALL), + ("some", _PAYS_SOME), + ("none", _PAYS_NONE), + ) + }, + "weighted_other_policyholders": float(weights[other].sum()), + "weighted_other_policyholders_with_employer_share": float( + weights[other & (raw > 0)].sum() + ), + "raw_employer_share_total": raw_universe_total, + "raw_anchor_universe_total": raw_anchor_total, + "raw_employer_share_mean_per_positive_person": ( + _weighted(weights, raw, raw_positive) / float(weights[raw_positive].sum()) + if raw_positive.any() + else 0.0 + ), + "raw_employer_share_band": list(_RAW_MEAN_BAND), + "raw_over_published_employer_mean": ( + _weighted(weights, raw, employer_pays) / published + if published + else float("nan") + ), + "scale_factor": scale, + "scale_factor_spread": ( + float(np.nanmax(ratio) - np.nanmin(ratio)) if raw_positive.any() else 0.0 + ), + "anchor_universe_employer_total": scale * raw_anchor_total, + "other_policyholder_employer_total": scale + * (raw_anchor_total - raw_universe_total), + "employed_share_of_anchor_universe": ( + raw_universe_total / raw_anchor_total if raw_anchor_total else float("nan") + ), + "employer_premium_by_tier": by_tier, + "employer_premium_by_sector": by_sector, + "employer_premium_outside_universe_rows": int((positive & ~universe).sum()), + "employer_premium_where_employer_pays_none_rows": int( + (positive & (codes.hipaid == _PAYS_NONE)).sum() + ), + "employer_premium_without_raw_share_rows": int( + (positive & ~raw_positive).sum() + ), + "meps_ic_private_active_employer_total": ( + meps_ic_private_active_employer_totals(load_meps_ic_esi_premium_cells()) + ), + } + return summary + + +def _missing_source_failure(missing: list[str], consequence: str) -> str: + return ( + f"assignment provenance incomplete: the person table lacks {missing}, so " + f"{consequence}. Rebuild the base with the meps_esi_premiums stage; a " + "release must carry the raw ASEC coverage columns the assignment read." + ) + + +def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: + """Structure and scale-invariant signal of the ESI employer premium input. + + Holds on any frame size (smoke builds included). It proves every positive + value sits in the employed-policyholder universe and equals one common + multiple of its MEPS-IC cell. Those proofs read the raw ASEC columns, so a + frame without them fails: it cannot be certified. + """ + + person = frame.table("person") + missing = [ + column + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="esi_premiums_signal", + passed=False, + failures=tuple(f"person column missing: {column}." for column in missing), + details={"missing": missing}, + ) + try: + summary = us_esi_premiums_summary(frame) + except SourceRuntimeError as exc: + return GateResult( + name="esi_premiums_signal", + passed=False, + failures=(str(exc),), + details={"structural_error": str(exc)}, + ) + failures: list[str] = [] + if summary["source_columns_missing"]: + failures.append( + _missing_source_failure( + list(summary["source_columns_missing"]), + "the universe, payment-status and MEPS-IC cell checks did not run", + ) + ) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + if person[column].dropna().nunique() < 2: + failures.append(f"{column}: constant column — zero mass or no signal.") + for key, label in ( + ("nonfinite_rows", "non-finite value(s)"), + ("negative_rows", "negative value(s)"), + ("clone_disagreement_source_persons", "support-clone disagreement(s)"), + ( + "employer_premium_outside_universe_rows", + "employer premium(s) outside employed policyholders", + ), + ( + "employer_premium_where_employer_pays_none_rows", + "employer premium(s) where the employer pays none", + ), + ( + "employer_premium_without_raw_share_rows", + "employer premium(s) with no MEPS-IC share", + ), + ): + count = int(summary.get(key, 0)) + if count: + failures.append(f"ESI premiums: {count} {label}.") + share = float(summary["employer_premium_positive_share"]) + low, high = _POSITIVE_SHARE_BAND + if not low <= share <= high: + failures.append( + f"employer premium positive share {share:.4f} outside [{low}, {high}]." + ) + if "raw_employer_share_mean_per_positive_person" in summary: + mean = float(summary["raw_employer_share_mean_per_positive_person"]) + low, high = _RAW_MEAN_BAND + if not low <= mean <= high: + failures.append( + f"raw MEPS-IC employer share per holder ${mean:,.0f} outside " + f"[${low:,.0f}, ${high:,.0f}]: the cell table's units are broken." + ) + # Relative to the factor: a float32 round trip leaves ~1e-7. + spread = float(summary["scale_factor_spread"]) + if spread > 1e-6 * float(summary["scale_factor"]): + failures.append( + f"employer premiums are not one common multiple of their MEPS-IC " + f"cells (scale-factor spread {spread:.3g})." + ) + return GateResult( + name="esi_premiums_signal", + passed=not failures, + failures=tuple(failures), + details=summary, + ) + + +def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult: + """Hold a release's weighted ESI employer premium total to its anchor. + + Runs on the calibrated export (dense and sparse). The anchor, BEA NIPA + 7.8 line 17, counts employer contributions for every policyholder, so the + gate compares the anchor-universe total: the column (employed + policyholders) plus the same scale factor applied to the other + policyholders' MEPS-IC shares, recomputed from the raw ASEC columns at the + frame's weights. It fails when the column is absent or zero-mass, when + the raw columns are absent, when that total leaves + ±``ANCHOR_RELATIVE_TOLERANCE`` of the anchor for ``time_period``, or when + the employed share of it leaves its band. CMS NHE Table 24 and MEPS-IC's + own private-sector enrollment total are recorded beside the verdict, not + gated. + """ + + person = frame.table("person") + missing = [ + column + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="esi_premiums_anchor", + passed=False, + failures=tuple(f"person column missing: {column}." for column in missing), + details={"missing": missing}, + ) + try: + summary = us_esi_premiums_summary(frame) + except SourceRuntimeError as exc: + return GateResult( + name="esi_premiums_anchor", + passed=False, + failures=(str(exc),), + details={"structural_error": str(exc)}, + ) + employer_total = float(summary["employer_premium_total"]) + year = str(int(time_period)) + failures: list[str] = [] + details: dict[str, object] = { + "time_period": int(time_period), + "employer_premium_total": employer_total, + "employer_premium_positive_persons": summary[ + "employer_premium_positive_persons" + ], + "employer_premium_mean_per_positive_person": summary[ + "employer_premium_mean_per_positive_person" + ], + "relative_tolerance": ANCHOR_RELATIVE_TOLERANCE, + "anchor": dict(EMPLOYER_PREMIUM_ANCHOR), + "cross_check": dict(EMPLOYER_PREMIUM_CROSS_CHECK), + } + if not employer_total > 0: + failures.append(f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: zero weighted mass.") + anchor = EMPLOYER_PREMIUM_ANCHOR["values"].get(year) + if anchor is None: + failures.append(f"no BEA NIPA 7.8 line 17 anchor for {year}.") + if summary["source_columns_missing"]: + failures.append( + _missing_source_failure( + list(summary["source_columns_missing"]), + "the anchor-universe total (every policyholder) cannot be computed", + ) + ) + else: + universe_total = float(summary["anchor_universe_employer_total"]) + employed_share = float(summary["employed_share_of_anchor_universe"]) + details |= { + key: summary[key] + for key in ( + "anchor_universe_employer_total", + "other_policyholder_employer_total", + "employed_share_of_anchor_universe", + "scale_factor", + "weighted_policyholders", + "weighted_employed_policyholders", + "weighted_other_policyholders", + "employer_premium_by_sector", + ) + } + if not np.isfinite(universe_total): + failures.append( + "the anchor-universe employer total is not finite: no employed " + "policyholder carries a MEPS-IC share to recover the scale from." + ) + elif anchor is not None: + relative = universe_total / float(anchor) - 1.0 + details |= {"anchor_value": float(anchor), "relative_error": relative} + if abs(relative) > ANCHOR_RELATIVE_TOLERANCE: + failures.append( + "employer premiums over every policyholder total " + f"${universe_total / 1e9:,.1f}B, {relative:+.1%} from BEA NIPA 7.8 " + f"line 17 ${float(anchor) / 1e9:,.1f}B ({year}); tolerance " + f"±{ANCHOR_RELATIVE_TOLERANCE:.0%}." + ) + low, high = _EMPLOYED_SHARE_BAND + details["employed_share_band"] = [low, high] + if not low <= employed_share <= high: + failures.append( + f"employed policyholders carry {employed_share:.4f} of the " + f"anchor-universe employer total, outside [{low}, {high}]." + ) + nhe = EMPLOYER_PREMIUM_CROSS_CHECK["values"].get(year) + if nhe is not None and np.isfinite(universe_total): + details["cross_check_ratio"] = universe_total / float(nhe) + active = dict(summary["meps_ic_private_active_employer_total"]) + if year in active: + private = float(dict(summary["employer_premium_by_sector"])["private"]) + details["private_active_cross_check"] = { + "meps_ic_private_active_employer_total": active[year], + "employer_premium_private_sector": private, + "ratio": private / active[year], + } + return GateResult( + name="esi_premiums_anchor", + passed=not failures, + failures=tuple(failures), + details=details, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py b/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py index ce76b43846..10cd209e3d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py @@ -47,6 +47,9 @@ US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS, ) from microcosm.build.us_runtime.energy_subsidy import US_ENERGY_SUBSIDY_OUTPUT_COLUMNS +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, +) from microcosm.build.us_runtime.farm_business_income import ( US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, ) @@ -189,6 +192,7 @@ *US_QBI_NONCONSTANT_PERSON_COLUMNS, *US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS, *US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, + *US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, *US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS, *US_ADULT_CARE_OUTPUT_COLUMNS, *US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py index 36405d1b4f..646b04d968 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py @@ -44,6 +44,7 @@ from microcosm.build.us_runtime.energy_subsidy import ( US_ENERGY_SUBSIDY_OUTPUT_COLUMNS, ) +from microcosm.build.us_runtime.esi_premiums import US_ESI_PREMIUMS_OUTPUT_COLUMNS from microcosm.build.us_runtime.geography_ladder import US_GEOGRAPHY_LADDER_COLUMNS from microcosm.build.us_runtime.hours_worked import ( US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, @@ -320,6 +321,11 @@ "retirement_distributions": { "person": frozenset(US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS), }, + # Base-build only (microcosm#454): no pool operator produces these yet, so + # a raw source frame that already carries them was staged elsewhere. + "esi_premiums": { + "person": frozenset(US_ESI_PREMIUMS_OUTPUT_COLUMNS), + }, "immigration": { "person": frozenset(US_IMMIGRATION_OUTPUT_COLUMNS), }, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index e803bb0137..38b4ecf8dc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -70,6 +70,7 @@ US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, ) from microcosm.build.us_runtime.energy_subsidy import US_ENERGY_SUBSIDY_OUTPUT_COLUMNS +from microcosm.build.us_runtime.esi_premiums import US_ESI_EMPLOYER_PREMIUM_COLUMN from microcosm.build.us_runtime.farm_business_income import ( US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, ) @@ -209,6 +210,7 @@ *US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, *US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS, *US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, + US_ESI_EMPLOYER_PREMIUM_COLUMN, *US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, *US_FORM_4952_OUTPUT_COLUMNS, *US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py index a67869a0e8..b9247bc63c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py @@ -42,6 +42,9 @@ derive_us_energy_subsidy_from_manifest, impute_us_energy_subsidy_to_puf_support_from_manifest, ) +from microcosm.build.us_runtime.esi_premiums import ( + derive_us_employer_esi_premiums_from_manifest, +) from microcosm.build.us_runtime.hours_worked import ( derive_us_hours_worked_from_manifest, ) @@ -315,6 +318,9 @@ def us_source_operation_handlers() -> Mapping[str, SourceOperationHandler]: derive_us_other_health_insurance_from_manifest ), "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, + "derive_employer_sponsored_insurance_premiums": ( + derive_us_employer_esi_premiums_from_manifest + ), "derive_education_inputs": derive_us_education_inputs_from_manifest, "derive_hours_worked": derive_us_hours_worked_from_manifest, "derive_medicare_take_up": derive_us_medicare_take_up_from_manifest, diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 7be8062b02..3a0dd0e3d9 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,21 +82,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_184 - assert fields["authored_normative_field_count"] == 32_404 + assert fields["configuration_field_count"] == 42_217 + assert fields["authored_normative_field_count"] == 32_437 assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_184 + assert fields["consumed_field_count"] == 42_217 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_010, + "legacy_behavior": 14_043, "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_498, + "legacy_behavior": 38_531, "no_generation0_effect": 3_686, } diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py b/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py new file mode 100644 index 0000000000..fcca559c7a --- /dev/null +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py @@ -0,0 +1,82 @@ +"""Engine contract of the ESI employer premium input (microcosm #454). + +``docs/us-esi-employer-premiums.md`` and the ``meps_esi_premiums`` stage notes +say how PolicyEngine-US consumes the input. These tests compute it in the +locked engine, so an engine release that reroutes it fails here rather than +silently changing what the stage's column means. +""" + +from __future__ import annotations + +import pytest + +from microcosm.build.us_runtime import US_ESI_EMPLOYER_PREMIUM_COLUMN + +PERIOD = 2024 +WAGES = 60_000.0 +MEASURES = ( + "irs_employment_income", + "payroll_tax_gross_wages", + "adjusted_gross_income", + "income_tax", + "employee_social_security_tax", + "cbo_household_market_income", +) + + +def _totals(**person_inputs: float) -> dict[str, float]: + from policyengine_us import Simulation + + person = {"age": {PERIOD: 40}, "employment_income": {PERIOD: WAGES}} + person |= {name: {PERIOD: value} for name, value in person_inputs.items()} + simulation = Simulation( + situation={ + "people": {"worker": person}, + "tax_units": {"tax_unit": {"members": ["worker"]}}, + "spm_units": {"spm_unit": {"members": ["worker"]}}, + "families": {"family": {"members": ["worker"]}}, + "marital_units": {"marital_unit": {"members": ["worker"]}}, + "households": { + "household": {"members": ["worker"], "state_code": {PERIOD: "TX"}} + }, + } + ) + return { + measure: float(simulation.calculate(measure, PERIOD).sum()) + for measure in MEASURES + } + + +@pytest.fixture(scope="module") +def baseline() -> dict[str, float]: + return _totals() + + +def test_the_input_is_a_person_year_input_leaf() -> None: + from policyengine_us import CountryTaxBenefitSystem + + variable = CountryTaxBenefitSystem().variables[US_ESI_EMPLOYER_PREMIUM_COLUMN] + assert variable.is_input_variable() + assert variable.entity.key == "person" + assert str(variable.definition_period).lower() == "year" + + +def test_the_engine_parameter_list_names_the_input() -> None: + from policyengine_us import CountryTaxBenefitSystem + + parameters = CountryTaxBenefitSystem().parameters + assert US_ESI_EMPLOYER_PREMIUM_COLUMN in ( + parameters.gov.household.cbo_market_income_additions(f"{PERIOD}-01-01") + ) + + +def test_an_employer_premium_moves_only_cbo_household_market_income(baseline) -> None: + premium = 10_000.0 + with_premium = _totals(**{US_ESI_EMPLOYER_PREMIUM_COLUMN: premium}) + + assert with_premium["cbo_household_market_income"] - baseline[ + "cbo_household_market_income" + ] == pytest.approx(premium) + for measure in MEASURES: + if measure != "cbo_household_market_income": + assert with_premium[measure] == pytest.approx(baseline[measure]), measure diff --git a/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py b/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py index 3717bc36bc..43eea6d2c3 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py @@ -315,7 +315,7 @@ def test_manifest_header_counts_match_parsed_families(self) -> None: manifest.reviewed_exclusions ) assert len(manifest.compiled_families) == 32 - assert len(manifest.reviewed_exclusions) == 52 + assert len(manifest.reviewed_exclusions) == 53 def test_wired_nipa_and_liheap_families_are_compiled(self) -> None: manifest = load_target_parity_manifest() diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index c5ec18bb51..79d903c887 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_184 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_217 assert field_ledger.source_counts == { - "authored": 32_404, + "authored": 32_437, "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_010, + "legacy_behavior": 14_043, "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_498, + "legacy_behavior": 38_531, "no_generation0_effect": 3_686, } - assert len({field.pointer for field in field_ledger.fields}) == 42_184 + assert len({field.pointer for field in field_ledger.fields}) == 42_217 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index d5ba0c21f6..6f3c4c4a1e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "32305247436ed439181e8b5a667343436a4b5622bff0402d2f13cc08a51e2bfc" + "707b3049f8b41615f14957f2820ce55d05a9df8a1df436bc29d63a50caeac1ea" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py index 4024b3a260..3c33a670ee 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py @@ -2099,3 +2099,49 @@ def test_concentration_limitation_is_true_to_the_penalty_solved() -> None: assert "'chi_square'" in penalized["reason"] assert "'softmax'" in penalized["reason"] assert "l2_lambda=0 " not in penalized["reason"] + + +def test_a_null_esi_premium_input_is_refused_before_any_default_fill(tmp_path) -> None: + """microcosm#454: an ACS-spine null in the ESI premium input is never a zero. + + The ``meps_esi_premiums`` stage runs on the ASEC donor rows only. A pooled + frame therefore carries the input as NaN on its ACS rows, and the + reviewed-null register would default-fill it to zero for half the + population while input coverage reads green. The refusal comes first: it + needs neither the engine nor the register file. + """ + + from microcosm.build.source_runtime import SourceRuntimeError + from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + module = _load_tool_module() + ids = np.arange(1, 5) + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_tax_unit_id": ids, + "person_spm_unit_id": ids, + "person_family_id": ids, + "person_marital_unit_id": ids, + # Two ASEC donor rows, two ACS rows the stage never saw. + "employer_sponsored_insurance_premiums": [9_000.0, 0.0, np.nan, np.nan], + } + ) + frame = Frame( + { + "person": person, + "household": pd.DataFrame({"household_id": ids}), + "tax_unit": pd.DataFrame({"tax_unit_id": ids}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids}), + "family": pd.DataFrame({"family_id": ids}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids}), + }, + US_SCHEMA, + {"household": Weights(values=np.ones(4), kind=WeightKind.DESIGN)}, + ) + absent_register = tmp_path / "no_such_summary.json" + with pytest.raises(SourceRuntimeError, match="never filled") as error: + module.fill_reviewed_nulls(frame, absent_register) + assert "ACS local release" in str(error.value) + assert "'employer_sponsored_insurance_premiums': 2" in str(error.value) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py b/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py index ecb0362987..2fad301645 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py @@ -68,6 +68,12 @@ } ) +#: The six columns microcosm #454 restores for the ``meps_esi_premiums`` stage. +#: No pinned H5 carries them, and they were never part of the #720 offline fix. +_ISSUE_454_ADDED = frozenset( + {"NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2", "PEMLR", "NOEMP"} +) + #: H5 row order differs from the member's, so a positional copy would fail. _H5_ORDER = [3, 0, 7, 5, 1, 6, 2, 4] @@ -367,7 +373,9 @@ def test_the_review_places_every_offline_fix_column_exactly_once(): not_restored = set(ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED) assert len(restored) == len(ASEC_CENSUS_PERSON_COLUMN_NAMES) assert restored.isdisjoint(not_restored) - assert restored | not_restored == _RECEIPT_720_ADDED + assert restored | not_restored == _RECEIPT_720_ADDED | _ISSUE_454_ADDED + assert _ISSUE_454_ADDED <= restored + assert _ISSUE_454_ADDED.isdisjoint(_RECEIPT_720_ADDED) assert not restored & set(ASEC_CENSUS_PERSON_IDENTITY_COLUMNS) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index a780c4814b..89d96e39ef 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf", + "sha256": "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py index 3f9bd565b0..3a572c7107 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py @@ -31,8 +31,8 @@ _TEST_PATHS = paths_for("microcosm-build") _ROOT = _TEST_PATHS.repository -_CHRONICLE_COMMIT = "51aa40fd28e2ca81937752515e773d123fecfb71" -_FACTS_SHA256 = "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac" +_CHRONICLE_COMMIT = "cf9ece2e7be11158c21fdc4dd38b44d92db552f6" +_FACTS_SHA256 = "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57" _SCHEMA_SHA256 = "bdb51e2a8115634633ba7448c4005930fd9c0bfbade5e1b079b6bc24da485d3d" @@ -55,7 +55,7 @@ def test_pin_records_the_rebuilt_bare_feed() -> None: assert pin.source_repo == "PolicyEngine/chronicle" assert pin.source_commit == _CHRONICLE_COMMIT assert pin.scope == "us_fiscal_targets" - assert pin.fact_row_count == 40178 + assert pin.fact_row_count == 40181 assert pin.facts_sha256 == _FACTS_SHA256 assert pin.consumer_fact_schema_versions == ("chronicle.consumer_fact.v3",) assert pin.consumer_fact_schema_sha256 == _SCHEMA_SHA256 @@ -128,7 +128,7 @@ def test_parity_resources_and_generator_restate_the_pin() -> None: assert manifest["reference"]["feed"] == generator.DEFAULT_FEED_NAME assert generator.DEFAULT_FEED_PATH.name == generator.DEFAULT_FEED_NAME assert manifest["reference"]["compiled_families"] == "32" - assert manifest["reference"]["reviewed_exclusions"] == "52" + assert manifest["reference"]["reviewed_exclusions"] == "53" def test_scope_names_every_pinned_pair_once() -> None: @@ -137,8 +137,9 @@ def test_scope_names_every_pinned_pair_once() -> None: scope = builder.load_scope(scope_path) raw = scope["raw"] assert scope["commit"] == load_us_chronicle_feed().source_commit - assert len(scope["pairs"]) == 637 == raw["pair_count"] - assert sum(len(p) for p in scope["runs"].values()) == 63 + assert len(scope["pairs"]) == 640 == raw["pair_count"] + # 63 package runs, plus the B4923C package in its three years (#454). + assert sum(len(p) for p in scope["runs"].values()) == 66 assert set(scope["runs"]) == { 2020, 2021, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py new file mode 100644 index 0000000000..24623916ae --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py @@ -0,0 +1,1678 @@ +"""US ESI premium stage tests (microcosm #454). + +Invariants the stage and its gates must hold for every valid input: + +* **Conservation**: the weighted employer share over every policyholder (the + column plus the other policyholders' share at the same scale factor) equals + the BEA NIPA 7.8 line 17 anchor of the build year, whatever the frame. The + column total is the anchor times the employed policyholders' share of the + weighted raw mass, so it equals the anchor exactly when no one outside the + column carries an employer share. +* **Monotonicity**: pricing a policyholder outside the column never raises a + worker's premium. +* **Bounds**: the output is finite and nonnegative. +* **Structural zeros**: no employer premium outside employed policyholders + with an employer, nor where the employer pays none. +* **Proportionality**: every employer premium is the same multiple of its + MEPS-IC cell share. +* **Cell reconciliation**: a cohort with MEPS-IC's own pays-all / pays-some + mix reproduces the cell's published employer mean. +* **Determinism and order invariance**: values depend on the person and the + frame's weighted raw mass, never on the seed or row order; support clones + agree. +* **Weight homogeneity**: scaling every weight by ``c`` leaves the total at + the anchor and scales each person's value by ``1 / c``. + +A scalar, per-person reference implementation of the declared rules is run +against the vectorized stage on generated frames (differential test). +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from importlib.resources import files + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from microcosm.build.source_runtime import SourceRuntimeError +from microcosm.build.us_runtime import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, + US_ESI_PREMIUMS_OUTPUT_COLUMNS, + US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, + US_ESI_PREMIUMS_STAGE_NAME, + US_SOURCE_MANIFEST, + derive_us_employer_esi_premiums_from_manifest, + load_meps_ic_esi_premium_cells, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + us_esi_premiums_stage_spec, + us_esi_premiums_summary, + with_us_esi_premium_inputs, +) +from microcosm.build.us_runtime import esi_premiums as esi +from microcosm.build.us_runtime.asec_census_person_columns import ( + ASEC_CENSUS_PERSON_COLUMNS, +) +from microcosm.build.us_runtime.esi_premiums import ( + meps_ic_private_active_employer_totals, + refuse_unassigned_us_esi_premiums, +) +from microcosm.build.us_runtime.release_input_coverage import ( + us_release_input_coverage_required_columns, + us_release_input_coverage_reviewed_exclusions, + us_release_reform_coverage_probes, +) +from microcosm.build.us_runtime.source_runtime import us_source_operation_handlers +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from test_support.paths import paths_for + +TIME_PERIOD = 2024 +#: BEA NIPA Table 7.8 line 17 (B4923C), CY2024, 2026-09-30 vintage: the scale +#: anchor (decision d1317). +ANCHOR = 977_034e6 +#: CMS NHE Table 24 employer contribution, CY2024: the recorded cross-check. +NHE = 1_047_000e6 +EMPLOYER = US_ESI_EMPLOYER_PREMIUM_COLUMN +_REPOSITORY_ROOT = paths_for("microcosm-build").repository +_CELLS = load_meps_ic_esi_premium_cells() +_STATES = sorted(int(state) for state in _CELLS["state_census_division"]) + +#: An employed private-sector self-only policyholder in a large California +#: firm whose employer pays some of the premium. +_BASE_ROW = { + "NOW_OWNGRP": 1, + "NOW_HIPAID": 2, + "NOW_GRPFTYP": 2, + "NOW_GRPFTYP2": 3, + "PEMLR": 1, + "NOEMP": 6, + "PEIO1COW": 4, + "state_fips": 6, +} +_NO_COVERAGE = {"NOW_OWNGRP": 2, "NOW_HIPAID": 0, "NOW_GRPFTYP": 0, "NOW_GRPFTYP2": 0} + + +def _frame( + rows: list[dict], + *, + weights: list[float] | np.ndarray | None = None, + state_on_household: bool = True, +) -> Frame: + """One household per person; State lives on the household table.""" + + records = [] + for index, row in enumerate(rows): + record = dict(_BASE_ROW) + record.update(row) + record.setdefault("person_id", index + 1) + record.setdefault("source_year", 2024) + record.setdefault("source_household_id", index + 1) + record.setdefault("source_person_id", 1) + records.append(record) + person = pd.DataFrame(records) + count = len(person) + ids = np.arange(1, count + 1, dtype="int64") + person["person_household_id"] = ids + person["person_tax_unit_id"] = ids + 1_000_000 + person["person_spm_unit_id"] = ids + 2_000_000 + person["person_family_id"] = ids + 3_000_000 + person["person_marital_unit_id"] = ids + 4_000_000 + household = pd.DataFrame({"household_id": ids}) + if state_on_household: + household["state_fips"] = person.pop("state_fips").to_numpy() + values = np.ones(count) if weights is None else np.asarray(weights, dtype=float) + return Frame( + { + "person": person, + "household": household, + "tax_unit": pd.DataFrame({"tax_unit_id": ids + 1_000_000}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids + 2_000_000}), + "family": pd.DataFrame({"family_id": ids + 3_000_000}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids + 4_000_000}), + }, + US_SCHEMA, + {"household": Weights(values=values, kind=WeightKind.DESIGN)}, + ) + + +def _run(rows, *, weights=None, seed: int = 0, time_period: int = TIME_PERIOD) -> Frame: + return with_us_esi_premium_inputs( + _frame(rows, weights=weights), seed=seed, time_period=time_period + ) + + +def _values(frame: Frame, column: str) -> np.ndarray: + return frame.table("person")[column].to_numpy(dtype=float) + + +def _weights(frame: Frame) -> np.ndarray: + return np.asarray(frame.resolve_weights("person").values, dtype=float) + + +def _private(tier: str, measure: str, state: int, size: str) -> float: + return _CELLS["private_state_2025"][tier][measure]["rows"][f"{state:02d}"][size] + + +def _no_contribution(tier: str, size: str) -> float: + return _CELLS["no_contribution_share_2025"][tier][size] / 100.0 + + +def _column_total(frame: Frame) -> float: + return float(_weights(frame) @ _values(frame, EMPLOYER)) + + +def _reweighted(frame: Frame, weights: np.ndarray) -> Frame: + """The same people at different household weights (a calibrated export).""" + + return Frame( + {entity: frame.table(entity) for entity in frame.entities}, + US_SCHEMA, + {"household": Weights(values=np.asarray(weights), kind=WeightKind.DESIGN)}, + ) + + +def _realistic_rows(count: int = 400) -> list[dict]: + """A mix whose weighted shares sit inside the signal-gate bands.""" + + rows: list[dict] = [] + for index in range(count): + state = _STATES[index % len(_STATES)] + kind = index % 20 + if kind < 4: # 20% employed policyholders, employer pays some + tier = (index // 20) % 3 + 1 + rows.append( + { + "NOW_GRPFTYP2": tier, + "NOW_GRPFTYP": 1 if tier in (1, 2) else 2, + "NOEMP": (index // 7) % 7, + "PEIO1COW": (1, 2, 3, 4, 5, 6)[(index // 3) % 6], + "state_fips": state, + } + ) + elif kind == 4: # employer pays all + rows.append({"NOW_HIPAID": 1, "state_fips": state}) + elif kind == 5: # retiree policyholder: priced, outside the column + rows.append({"PEMLR": 5, "PEIO1COW": 0, "NOEMP": 0, "state_fips": state}) + else: # no employment-based coverage of their own + rows.append( + { + **_NO_COVERAGE, + "PEMLR": (0, 1, 5, 7)[index % 4], + "state_fips": state, + } + ) + return rows + + +# --- the manifest contract --------------------------------------------------- + + +class TestManifestDeclaration: + def test_stage_declares_its_output_and_the_reviewed_operations(self) -> None: + spec = us_esi_premiums_stage_spec() + assert spec.stage == US_ESI_PREMIUMS_STAGE_NAME == "meps_esi_premiums" + assert tuple(spec.outputs) == US_ESI_PREMIUMS_OUTPUT_COLUMNS + assert US_ESI_PREMIUMS_OUTPUT_COLUMNS == (EMPLOYER,) + assert US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS == (EMPLOYER,) + assert [operation.kind for operation in spec.operations] == [ + "read_table", + "derive_employer_sponsored_insurance_premiums", + ] + assert spec is not None and US_SOURCE_MANIFEST.stage_map()[spec.stage] + + def test_handlers_are_registered(self) -> None: + handlers = us_source_operation_handlers() + assert ( + handlers["derive_employer_sponsored_insurance_premiums"] + is derive_us_employer_esi_premiums_from_manifest + ) + # The pre-tax employee premium is not this stage's: the engine treats + # it as disjoint from the reported premium (policyengine-us#10046). + assert "derive_pre_tax_health_insurance_premiums" not in handlers + + def test_manifest_pins_the_cells_the_anchor_and_every_source_pdf(self) -> None: + spec = us_esi_premiums_stage_spec() + employer = spec.operations[1].parameters + assert employer["cells_sha256"] == esi._CELLS_SHA256 + assert employer["anchor"]["table"].startswith("NIPA Table 7.8") + assert employer["anchor"]["row"] == esi.EMPLOYER_PREMIUM_ANCHOR["row"] + assert "B4923C" in employer["anchor"]["row"] + assert employer["anchor"]["values"]["2024"] == ANCHOR + assert employer["anchor_universe"] == "NOW_OWNGRP == 1" + assert "PEMLR in (1, 2)" in employer["universe"] + # A zip member is addressed as "!". + pinned = { + artifact["locator"].split("!")[0]: artifact["sha256"] + for artifact in spec.artifacts + if artifact.get("sha256") + } + for source in _CELLS["sources"].values(): + assert pinned[source["url"]] == source["sha256"] + for pin in (esi.EMPLOYER_PREMIUM_ANCHOR, esi.EMPLOYER_PREMIUM_CROSS_CHECK): + assert pinned[pin["source"]] == pin["sha256"] + + def test_a_drifted_manifest_operation_is_refused(self, monkeypatch) -> None: + monkeypatch.setattr( + esi, + "_EMPLOYER_PREMIUM_PARAMETERS", + {**esi._EMPLOYER_PREMIUM_PARAMETERS, "tier_column": "NOW_GRPFTYP"}, + ) + with pytest.raises(ValueError, match="drifted"): + us_esi_premiums_stage_spec() + + def test_the_restored_census_columns_are_the_ones_the_stage_requires(self) -> None: + restored = {column.name: column for column in ASEC_CENSUS_PERSON_COLUMNS} + for name in ("NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2"): + assert restored[name].domain == esi._CODE_DOMAINS[name] + assert restored["PEMLR"].domain == esi._CODE_DOMAINS["PEMLR"] + assert restored["NOEMP"].domain == esi._CODE_DOMAINS["NOEMP"] + assert set(restored) >= set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) - { + "PEIO1COW", + "state_fips", + } + + +# --- the packaged MEPS-IC cells ----------------------------------------------- + + +class TestCells: + def test_digest_is_pinned(self, monkeypatch) -> None: + monkeypatch.setattr(esi, "_CELLS_SHA256", "0" * 64) + with pytest.raises(SourceRuntimeError, match="not the reviewed"): + load_meps_ic_esi_premium_cells() + + def test_national_rows_are_the_published_ahrq_figures(self) -> None: + rows = _CELLS["private_state_2025"] + assert rows["single"]["premium"]["rows"]["US"]["total"] == 9_025 + assert rows["family"]["premium"]["rows"]["US"]["total"] == 26_281 + assert rows["employee_plus_one"]["premium"]["rows"]["US"]["total"] == 17_901 + prior = _CELLS["private_national_2024"] + # MEPS-IC Research Findings #54 (2024). + assert prior["single"]["premium"]["total"] == 8_486 + assert prior["single"]["employee_contribution"]["total"] == 1_789 + assert prior["family"]["premium"]["total"] == 24_540 + assert prior["family"]["employee_contribution"]["total"] == 7_216 + assert prior["employee_plus_one"]["premium"]["total"] == 16_931 + assert prior["employee_plus_one"]["employee_contribution"]["total"] == 4_707 + + def test_every_state_and_division_has_a_cell(self) -> None: + divisions = _CELLS["state_census_division"] + assert len(divisions) == 51 + assert len(set(divisions.values())) == 9 + for tier in _CELLS["tiers"]: + for measure in ("premium", "employee_contribution"): + rows = _CELLS["private_state_2025"][tier][measure]["rows"] + assert set(rows) == {"US", *divisions} + public = _CELLS["public_division_2024"][tier][measure]["rows"] + assert set(public) == {"US", *set(divisions.values())} + + def test_suppressed_cells_are_the_nine_small_firm_contributions(self) -> None: + suppressed = { + (tier, measure, state, size) + for tier in _CELLS["tiers"] + for measure in ("premium", "employee_contribution") + for state, row in _CELLS["private_state_2025"][tier][measure][ + "rows" + ].items() + for size in ("total", "lt50", "50plus") + if row[size] is None + } + assert suppressed == { + ("employee_plus_one", "employee_contribution", state, "lt50") + for state in ("02", "21", "45", "46") + } | { + ("family", "employee_contribution", state, "lt50") + for state in ("02", "10", "35", "45", "54") + } + + def test_every_resolved_cell_leaves_a_positive_employer_share(self) -> None: + for tier in _CELLS["tiers"]: + for state in _CELLS["state_census_division"]: + for size in ("total", "lt50", "50plus"): + premium = esi._private_cell(_CELLS, tier, "premium", state, size) + paid = esi._private_cell( + _CELLS, tier, "employee_contribution", state, size + ) + assert 0 < paid < premium < 60_000 + # The contribution of an employee who pays one still + # leaves the employer a share. + assert paid / (1 - _no_contribution(tier, size)) < premium + for division in set(_CELLS["state_census_division"].values()): + for column in ("state", "all_state_and_local"): + premium = esi._government_cell( + _CELLS, tier, "premium", division, column + ) + paid = esi._government_cell( + _CELLS, tier, "employee_contribution", division, column + ) + assert 0 < paid < premium < 60_000 + assert paid / (1 - _no_contribution(tier, "50plus")) < premium + + def test_no_contribution_shares_are_the_published_national_rows(self) -> None: + # MEPS-IC 2025 Tables II.C.4.a, II.D.4.a and II.E.4.a, United States. + published = { + "single": {"total": 13.0, "lt50": 35.5, "50plus": 8.7}, + "family": {"total": 7.7, "lt50": 28.6, "50plus": 4.6}, + "employee_plus_one": {"total": 6.0, "lt50": 23.1, "50plus": 3.6}, + } + for tier, row in published.items(): + table = _CELLS["no_contribution_share_2025"][tier] + assert {size: table[size] for size in row} == row + assert "required no" in table["title"] + + def test_private_enrollment_rows_price_the_active_employee_total(self) -> None: + totals = meps_ic_private_active_employer_totals(_CELLS) + rows = _CELLS["private_enrollment_national"]["2024"] + assert rows["employees"]["total"] == 139_441_077 + assert rows["offer_percent"]["total"] == 85.1 + assert rows["enrolled_percent"]["total"] == 55.3 + enrolled = 139_441_077 * 0.851 * 0.553 + by_hand = enrolled * ( + 0.587 * (8_486 - 1_789) + + 0.237 * (24_540 - 7_216) + + 0.176 * (16_931 - 4_707) + ) + assert totals["2024"] == pytest.approx(by_hand, rel=1e-12) + assert totals["2024"] == pytest.approx(668.6e9, rel=5e-4) + assert totals["2025"] == pytest.approx(714.4e9, rel=5e-4) + for year in ("2024", "2025"): + shares = _CELLS["private_enrollment_national"][year]["tier_share_percent"] + assert sum(shares[tier]["total"] for tier in _CELLS["tiers"]) == ( + pytest.approx(100.0) + ) + + +# --- the employer premium ------------------------------------------------------- + + +class TestEmployerPremium: + def test_weighted_total_over_every_policyholder_is_the_anchor(self) -> None: + result = _run(_realistic_rows(), weights=np.linspace(50, 5_000, 400)) + summary = us_esi_premiums_summary(result) + assert summary["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + # The retiree policyholders are priced, so the column is the employed + # policyholders' part of the anchor, not all of it. + share = summary["employed_share_of_anchor_universe"] + assert 0 < share < 1 + assert _column_total(result) == pytest.approx(ANCHOR * share, rel=1e-12) + assert summary["other_policyholder_employer_total"] == pytest.approx( + ANCHOR * (1 - share), rel=1e-9 + ) + + def test_without_other_policyholders_the_column_is_the_whole_anchor(self) -> None: + rows = [{"NOW_HIPAID": 1}, {"NOW_HIPAID": 2}, dict(_NO_COVERAGE)] + assert _column_total(_run(rows)) == pytest.approx(ANCHOR, rel=1e-12) + + def test_a_priced_retiree_takes_their_share_out_of_the_column(self) -> None: + # A worker and a retiree whose former employer pays the whole premium. + # The retiree has no employer class, so they take California's + # all-sizes private cell; the worker takes the 50-or-more cell. + retiree = {"NOW_HIPAID": 1, "PEMLR": 5, "PEIO1COW": 0, "NOEMP": 0} + alone = _values(_run([{"NOW_HIPAID": 1}]), EMPLOYER) + result = _run([{"NOW_HIPAID": 1}, retiree]) + together = _values(result, EMPLOYER) + worker = _private("single", "premium", 6, "50plus") + retired = _private("single", "premium", 6, "total") + assert alone[0] == pytest.approx(ANCHOR) + assert together[1] == 0 + assert together[0] == pytest.approx(ANCHOR * worker / (worker + retired)) + summary = us_esi_premiums_summary(result) + assert summary["other_policyholder_employer_total"] == pytest.approx( + ANCHOR * retired / (worker + retired) + ) + assert summary["weighted_other_policyholders_with_employer_share"] == 1 + + def test_a_non_employed_policyholder_keeps_the_cell_of_a_reported_employer( + self, + ) -> None: + # Unemployed, last job with a State government in Texas, small NOEMP: + # the State-government division cell, whatever the firm size. + former = {"NOW_HIPAID": 1, "PEMLR": 4, "PEIO1COW": 2, "NOEMP": 1} + result = _run([{"NOW_HIPAID": 1}, {**former, "state_fips": 48}]) + summary = us_esi_premiums_summary(result) + published = _CELLS["public_division_2024"]["single"]["premium"]["rows"][ + "West South Central" + ]["state"] + aged = published * 9_025 / 8_486 + assert summary["other_policyholder_employer_total"] / _column_total( + result + ) == pytest.approx(aged / _private("single", "premium", 6, "50plus")) + + def test_payment_status_sets_the_share_of_the_cell(self) -> None: + rows = [{"NOW_HIPAID": code} for code in (1, 2, 3)] + employer = _values(_run(rows), EMPLOYER) + premium = _private("single", "premium", 6, "50plus") + paid = _private("single", "employee_contribution", 6, "50plus") + # The published average contribution counts the 8.7% who pay nothing. + paying = paid / (1 - 0.087) + assert _no_contribution("single", "50plus") == 0.087 + assert employer[2] == 0 + assert employer[0] / employer[1] == pytest.approx(premium / (premium - paying)) + # One scale factor: both are the same multiple of their raw share. + assert employer[0] / premium == pytest.approx(employer[1] / (premium - paying)) + + @pytest.mark.parametrize("tier_code", [1, 2, 3]) + @pytest.mark.parametrize(("noemp", "size"), [(1, "lt50"), (4, "50plus")]) + @pytest.mark.parametrize("state", [6, 36, 48]) + def test_the_published_mix_reproduces_the_published_employer_mean( + self, tier_code, noemp, size, state + ) -> None: + tier = esi._TIER_BY_CODE[tier_code] + share = _no_contribution(tier, size) + cell = { + "NOW_GRPFTYP2": tier_code, + "NOW_GRPFTYP": 1 if tier_code in (1, 2) else 2, + "NOEMP": noemp, + "state_fips": state, + } + # MEPS-IC's own mix: `share` of enrollees pay nothing, the rest pay. + result = _run( + [{**cell, "NOW_HIPAID": 1}, {**cell, "NOW_HIPAID": 2}], + weights=[share, 1 - share], + ) + summary = us_esi_premiums_summary(result) + assert summary["raw_over_published_employer_mean"] == pytest.approx(1.0) + published = _private(tier, "premium", state, size) - esi._private_cell( + _CELLS, tier, "employee_contribution", f"{state:02d}", size + ) + modeled = _column_total(result) / summary["scale_factor"] + assert modeled == pytest.approx(published) + + def test_every_cell_reconciles_under_the_published_mix(self) -> None: + """Exhaustive: every tier x State x size private cell and every + tier x division x column government cell. + + For each cell, the no-contribution share of enrollees paying nothing + and the rest paying the conditional contribution average the cell's + premium less its average contribution. A suppressed contribution cell + reconciles to its fallback value, since AHRQ published none. + """ + + rows = [] + tiers = ((1, 1), (2, 1), (3, 2)) # NOW_GRPFTYP2, NOW_GRPFTYP + for tier_code, grpftyp in tiers: + for state in _STATES: + # Private by size (NOEMP 0, 1, 3), State and local government. + for sector, noemp in ((4, 0), (4, 1), (4, 3), (2, 0), (3, 0), (1, 0)): + for hipaid in (1, 2): + rows.append( + { + "NOW_GRPFTYP2": tier_code, + "NOW_GRPFTYP": grpftyp, + "NOW_HIPAID": hipaid, + "PEIO1COW": sector, + "NOEMP": noemp, + "state_fips": state, + } + ) + person = esi._person_with_state(_frame(rows)) + codes, raw, premium, contribution = esi._person_raw_shares(person) + _, _, share = esi._cell_values(codes, _CELLS) + pays_all, pays_some = raw[0::2], raw[1::2] + assert (codes.hipaid[0::2] == 1).all() and (codes.hipaid[1::2] == 2).all() + unpaid = share[0::2] + modeled = unpaid * pays_all + (1 - unpaid) * pays_some + published = premium[0::2] - contribution[0::2] + assert len(modeled) == 3 * 51 * 6 + assert (pays_some > 0).all() and (pays_some < pays_all).all() + np.testing.assert_allclose(modeled, published, rtol=1e-12) + + def test_the_unconditional_contribution_would_overstate_the_cell_mean(self) -> None: + # The national under-50 single-coverage cell: premium $9,034, average + # contribution $1,924, 35.5% of enrollees pay nothing. Subtracting the + # average from every payer gives $7,793, above the published $7,110. + rows = _CELLS["private_state_2025"]["single"] + premium = rows["premium"]["rows"]["US"]["lt50"] + paid = rows["employee_contribution"]["rows"]["US"]["lt50"] + share = _no_contribution("single", "lt50") + assert (premium, paid, share) == (9_034, 1_924, 0.355) + unconditional = share * premium + (1 - share) * (premium - paid) + conditional = share * premium + (1 - share) * (premium - paid / (1 - share)) + assert unconditional == pytest.approx(7_793.02) + assert conditional == pytest.approx(premium - paid) == pytest.approx(7_110) + + def test_only_employed_policyholders_with_an_employer_carry_a_premium(self) -> None: + rows = [ + {}, # the universe + dict(_NO_COVERAGE), # dependent or uncovered + {"NOW_OWNGRP": 0, "NOW_HIPAID": 0, "NOW_GRPFTYP": 0, "NOW_GRPFTYP2": 0}, + {"PEMLR": 5, "PEIO1COW": 0}, # retiree policyholder + {"PEMLR": 3}, # on layoff + {"PEMLR": 4}, # looking + {"PEMLR": 6}, # disabled + {"PEIO1COW": 7}, # self-employed, unincorporated + {"PEIO1COW": 8}, # without pay + {"PEMLR": 2}, # employed, absent + {"PEIO1COW": 6}, # self-employed, incorporated + ] + employer = _values(_run(rows), EMPLOYER) + assert (employer[[0, 9, 10]] > 0).all() + assert (employer[1:9] == 0).all() + + @pytest.mark.parametrize( + ("grpftyp", "grpftyp2", "tier"), + [(1, 1, "family"), (1, 2, "employee_plus_one"), (2, 3, "single")], + ) + def test_tier_selects_the_cell(self, grpftyp, grpftyp2, tier) -> None: + rows = [ + {"NOW_HIPAID": 1}, + {"NOW_HIPAID": 1, "NOW_GRPFTYP": grpftyp, "NOW_GRPFTYP2": grpftyp2}, + ] + employer = _values(_run(rows), EMPLOYER) + assert employer[1] / employer[0] == pytest.approx( + _private(tier, "premium", 6, "50plus") + / _private("single", "premium", 6, "50plus") + ) + + @pytest.mark.parametrize( + ("noemp", "size"), + [(0, "total"), (1, "lt50"), (2, "lt50"), (3, "50plus"), (5, "50plus")], + ) + def test_employer_size_and_state_select_the_private_cell(self, noemp, size) -> None: + rows = [{"NOW_HIPAID": 1}, {"NOW_HIPAID": 1, "NOEMP": noemp, "state_fips": 36}] + employer = _values(_run(rows), EMPLOYER) + assert employer[1] / employer[0] == pytest.approx( + _private("single", "premium", 36, size) + / _private("single", "premium", 6, "50plus") + ) + + @pytest.mark.parametrize( + ("sector", "column"), + [(1, "all_state_and_local"), (2, "state"), (3, "all_state_and_local")], + ) + def test_government_employers_take_the_aged_division_cell( + self, sector, column + ) -> None: + # Texas is West South Central; NOEMP must not matter for government. + rows = [ + {"NOW_HIPAID": 1}, + {"NOW_HIPAID": 1, "PEIO1COW": sector, "state_fips": 48, "NOEMP": 1}, + ] + employer = _values(_run(rows), EMPLOYER) + assert _CELLS["state_census_division"]["48"] == "West South Central" + published = _CELLS["public_division_2024"]["single"]["premium"]["rows"][ + "West South Central" + ][column] + aged = published * 9_025 / 8_486 + assert employer[1] / employer[0] == pytest.approx( + aged / _private("single", "premium", 6, "50plus") + ) + + def test_a_suppressed_cell_takes_the_national_cell_at_the_state_level(self) -> None: + # Alaska family small-firm employee contribution is suppressed. + family = {"NOW_GRPFTYP": 1, "NOW_GRPFTYP2": 1, "state_fips": 2, "NOEMP": 1} + rows = [{**family, "NOW_HIPAID": 1}, {**family, "NOW_HIPAID": 2}] + employer = _values(_run(rows), EMPLOYER) + contributions = _CELLS["private_state_2025"]["family"]["employee_contribution"][ + "rows" + ] + assert contributions["02"]["lt50"] is None + fallback = ( + contributions["US"]["lt50"] + * contributions["02"]["total"] + / contributions["US"]["total"] + ) + premium = _private("family", "premium", 2, "lt50") + paying = fallback / (1 - _no_contribution("family", "lt50")) + assert employer[1] / employer[0] == pytest.approx((premium - paying) / premium) + + def test_the_assignment_does_not_depend_on_the_seed(self) -> None: + rows = _realistic_rows(120) + first, second = _run(rows, seed=1), _run(rows, seed=2) + assert np.array_equal(_values(first, EMPLOYER), _values(second, EMPLOYER)) + + def test_state_may_already_sit_on_the_person_table(self) -> None: + rows = _realistic_rows(60) + on_household = with_us_esi_premium_inputs( + _frame(rows), seed=0, time_period=TIME_PERIOD + ) + on_person = with_us_esi_premium_inputs( + _frame(rows, state_on_household=False), seed=0, time_period=TIME_PERIOD + ) + assert np.array_equal( + _values(on_household, EMPLOYER), _values(on_person, EMPLOYER) + ) + + @pytest.mark.parametrize("year", [2023, 2024, 2025]) + def test_each_pinned_year_scales_to_its_own_anchor(self, year) -> None: + result = _run(_realistic_rows(60), time_period=year) + total = us_esi_premiums_summary(result)["anchor_universe_employer_total"] + assert total == pytest.approx( + esi.EMPLOYER_PREMIUM_ANCHOR["values"][str(year)], rel=1e-12 + ) + + +class TestRefusals: + @pytest.mark.parametrize( + "column", [c for c in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS] + ) + def test_a_missing_source_column_is_source_unavailable(self, column) -> None: + frame = _frame(_realistic_rows(40), state_on_household=False) + frame.table("person").drop(columns=[column], inplace=True) + with pytest.raises(SourceRuntimeError, match="source-unavailable"): + with_us_esi_premium_inputs(frame, seed=0, time_period=TIME_PERIOD) + + @pytest.mark.parametrize("column", ["NOW_OWNGRP", "NOW_HIPAID", "PEMLR", "NOEMP"]) + def test_a_pooled_nan_is_never_defaulted(self, column) -> None: + frame = _frame(_realistic_rows(40)) + person = frame.table("person") + person[column] = person[column].astype(float) + person.loc[3, column] = np.nan + with pytest.raises(SourceRuntimeError, match="missing or non-integer"): + with_us_esi_premium_inputs(frame, seed=0, time_period=TIME_PERIOD) + + @pytest.mark.parametrize( + ("column", "value"), + [("NOW_OWNGRP", 3), ("NOW_HIPAID", 4), ("NOW_GRPFTYP2", 4), ("NOEMP", 7)], + ) + def test_a_code_outside_the_census_codebook_is_refused(self, column, value) -> None: + with pytest.raises(SourceRuntimeError, match="outside the Census codebook"): + _run([{}, {column: value}]) + + @pytest.mark.parametrize( + ("row", "violation"), + [ + ({"NOW_HIPAID": 0}, "policyholder_without_payment_status"), + ({"NOW_GRPFTYP2": 0}, "policyholder_without_tier"), + ({**_NO_COVERAGE, "NOW_HIPAID": 2}, "non_policyholder_with_payment"), + ({**_NO_COVERAGE, "NOW_GRPFTYP2": 3}, "non_policyholder_with_tier"), + ({"NOW_GRPFTYP": 1, "NOW_GRPFTYP2": 3}, "grpftyp_disagrees"), + ], + ) + def test_the_census_universe_relations_are_enforced(self, row, violation) -> None: + with pytest.raises(SourceRuntimeError, match=violation): + _run([{}, row]) + + @pytest.mark.parametrize( + "other", + [ + dict(_NO_COVERAGE), + # A priced retiree does not give the column any mass to scale. + {"NOW_HIPAID": 1, "PEMLR": 5, "PEIO1COW": 0}, + ], + ) + def test_a_frame_with_no_employer_paid_mass_cannot_be_scaled(self, other) -> None: + with pytest.raises(SourceRuntimeError, match="no weighted employer-paid"): + _run([{"NOW_HIPAID": 3}, other]) + + @pytest.mark.parametrize("year", [2022, 2026, 2030]) + def test_an_unpinned_build_year_is_refused(self, year) -> None: + # The pinned BEA workbook column covers 2023 to 2025. + with pytest.raises( + SourceRuntimeError, match=f"no BEA NIPA 7.8 line 17 anchor for {year}" + ): + _run([{}], time_period=year) + + def test_an_unknown_state_is_refused(self) -> None: + with pytest.raises(SourceRuntimeError, match="State FIPS 72"): + _run([{}, {"state_fips": 72}]) + + +# --- idempotence and support clones ----------------------------------------------- + + +def _cloned(frame: Frame) -> Frame: + """Two support clones of every person at half weight, same source identity.""" + + person = frame.table("person") + doubled = pd.concat([person, person], ignore_index=True) + count = len(person) + doubled["person_id"] = np.arange(1, 2 * count + 1) + for column, offset in ( + ("person_household_id", 0), + ("person_tax_unit_id", 1_000_000), + ("person_spm_unit_id", 2_000_000), + ("person_family_id", 3_000_000), + ("person_marital_unit_id", 4_000_000), + ): + doubled[column] = np.arange(1, 2 * count + 1) + offset + ids = np.arange(1, 2 * count + 1, dtype="int64") + household = pd.concat( + [frame.table("household"), frame.table("household")], ignore_index=True + ) + household["household_id"] = ids + weights = np.tile(frame.weights_for("household").values / 2.0, 2) + return Frame( + { + "person": doubled, + "household": household, + "tax_unit": pd.DataFrame({"tax_unit_id": ids + 1_000_000}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids + 2_000_000}), + "family": pd.DataFrame({"family_id": ids + 3_000_000}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids + 4_000_000}), + }, + US_SCHEMA, + {"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + + +class TestIdempotenceAndClones: + def test_a_frame_that_carries_the_inputs_passes_through_untouched(self) -> None: + once = _run(_realistic_rows()) + twice = with_us_esi_premium_inputs(once, seed=99, time_period=TIME_PERIOD) + assert twice is once + + def test_support_clones_conserve_mass_and_agree(self) -> None: + once = _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) + cloned = _cloned(once) + assert us_esi_premiums_signal_gate(cloned).passed + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + assert _weights(cloned) @ _values(cloned, column) == pytest.approx( + _weights(once) @ _values(once, column), rel=1e-12 + ) + assert us_esi_premiums_anchor_gate(cloned, time_period=TIME_PERIOD).passed + + def test_a_preexisting_surface_that_fails_the_gate_is_refused(self) -> None: + once = _run(_realistic_rows()) + person = once.table("person") + holder = int(np.flatnonzero(person[EMPLOYER].to_numpy() > 0)[0]) + person.loc[holder, EMPLOYER] *= 2 + with pytest.raises(SourceRuntimeError, match="fail the signal gate"): + with_us_esi_premium_inputs(once, seed=0, time_period=TIME_PERIOD) + + +# --- the stage signal gate -------------------------------------------------------- + + +class TestSignalGate: + def _staged(self) -> Frame: + return _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) + + def test_passes_on_a_staged_frame_and_reports_the_lineage(self) -> None: + gate = us_esi_premiums_signal_gate(self._staged()) + assert gate.passed, gate.failures + details = gate.details + column = details["employer_premium_total"] + assert details["source_columns_missing"] == [] + assert details["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + assert column == pytest.approx( + ANCHOR * details["employed_share_of_anchor_universe"], rel=1e-12 + ) + assert details["scale_factor"] == pytest.approx( + ANCHOR / details["raw_anchor_universe_total"] + ) + assert details["scale_factor_spread"] <= 1e-9 * details["scale_factor"] + assert details["cells_sha256"] == esi._CELLS_SHA256 + assert details["anchor"]["series"] == "B4923C" + assert details["cross_check"]["row"].startswith("Employer Contribution") + assert sum(details["employer_premium_by_tier"].values()) == pytest.approx( + column + ) + assert sum(details["employer_premium_by_sector"].values()) == pytest.approx( + column + ) + json.dumps(dict(details)) # the base summary serializes it + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_missing_column_fails(self, column) -> None: + frame = self._staged() + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert gate.failures == (f"person column missing: {column}.",) + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_zero_mass_column_fails(self, column) -> None: + frame = self._staged() + frame.table("person")[column] = 0.0 + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert any("constant column" in failure for failure in gate.failures) + + def _fails_with(self, mutate, match: str) -> None: + frame = self._staged() + mutate(frame.table("person")) + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert any(match in failure for failure in gate.failures), gate.failures + + def test_a_premium_outside_the_universe_fails(self) -> None: + def mutate(person): + outsider = int(np.flatnonzero(person["NOW_OWNGRP"].to_numpy() == 2)[0]) + person.loc[outsider, EMPLOYER] = 9_000.0 + + self._fails_with(mutate, "outside employed policyholders") + + def test_a_premium_where_the_employer_pays_none_fails(self) -> None: + frame = _run(_realistic_rows() + [{"NOW_HIPAID": 3}]) + frame.table("person").loc[400, EMPLOYER] = 5_000.0 + gate = us_esi_premiums_signal_gate(frame) + assert any("employer pays none" in failure for failure in gate.failures) + + def test_a_value_that_is_not_the_common_cell_multiple_fails(self) -> None: + def mutate(person): + holder = int(np.flatnonzero(person[EMPLOYER].to_numpy() > 0)[0]) + person.loc[holder, EMPLOYER] *= 1.01 + + self._fails_with(mutate, "not one common multiple") + + def test_a_negative_value_fails(self) -> None: + self._fails_with( + lambda person: person.__setitem__(EMPLOYER, -person[EMPLOYER]), + "negative value", + ) + + def test_clone_disagreement_fails(self) -> None: + cloned = _cloned(self._staged()) + person = cloned.table("person") + holder = int(np.flatnonzero(person[EMPLOYER].to_numpy() > 0)[0]) + person.loc[holder, EMPLOYER] = 0.0 + gate = us_esi_premiums_signal_gate(cloned) + assert any("support-clone disagreement" in f for f in gate.failures) + + def test_an_implausible_positive_share_fails(self) -> None: + gate = us_esi_premiums_signal_gate(_run([{} for _ in range(50)])) + assert any("positive share" in failure for failure in gate.failures) + + @staticmethod + def _scaled_cells(factor: float) -> dict: + scaled = json.loads(json.dumps(_CELLS)) + for tier in scaled["tiers"]: + for measure in ("premium", "employee_contribution"): + for row in scaled["private_state_2025"][tier][measure]["rows"].values(): + for size in ("total", "lt50", "50plus"): + if row[size] is not None: + row[size] *= factor + for row in scaled["public_division_2024"][tier][measure][ + "rows" + ].values(): + for column in row: + row[column] *= factor + return scaled + + def test_a_unit_error_in_the_cells_cannot_hide_behind_the_scale_factor( + self, monkeypatch + ) -> None: + # Dollars read as hundreds of dollars. + broken = self._scaled_cells(0.01) + monkeypatch.setattr(esi, "load_meps_ic_esi_premium_cells", lambda: broken) + frame = _run(_realistic_rows()) + total = us_esi_premiums_summary(frame)["anchor_universe_employer_total"] + assert total == pytest.approx(ANCHOR) # the scale factor hid the level + gate = us_esi_premiums_signal_gate(frame) + assert any("units are broken" in f for f in gate.failures) + + def test_a_level_error_inside_the_band_is_outside_this_gates_reach( + self, monkeypatch + ) -> None: + # The documented limit: every cell 20% low still averages inside the + # raw-mean band, the scale factor absorbs it, and this gate passes. A + # re-pinned table is reviewed against the PDFs, not caught here. + low = self._scaled_cells(0.8) + monkeypatch.setattr(esi, "load_meps_ic_esi_premium_cells", lambda: low) + gate = us_esi_premiums_signal_gate(_run(_realistic_rows())) + assert gate.passed, gate.failures + low_band, high_band = esi._RAW_MEAN_BAND + mean = gate.details["raw_employer_share_mean_per_positive_person"] + assert low_band <= mean <= high_band + + @pytest.mark.parametrize( + "column", + [c for c in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS if c != "state_fips"], + ) + def test_a_frame_without_a_raw_column_cannot_be_certified(self, column) -> None: + frame = self._staged() + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert gate.details["source_columns_missing"] == [column] + assert any("assignment provenance incomplete" in f for f in gate.failures) + assert "scale_factor" not in gate.details + + def test_a_misassignment_cannot_pass_by_dropping_a_raw_column(self) -> None: + # Reverse the premium vector: the weighted total is unchanged, the + # people who carry it are wrong. With the raw columns the gate names + # the misassignment; with one dropped it still fails, on provenance. + frame = _run(_realistic_rows()) # unit weights: the total is preserved + person = frame.table("person") + before = person[EMPLOYER].sum() + person[EMPLOYER] = person[EMPLOYER].to_numpy()[::-1].copy() + assert person[EMPLOYER].sum() == pytest.approx(before) + with_raw = us_esi_premiums_signal_gate(frame) + assert not with_raw.passed + assert any("outside employed policyholders" in f for f in with_raw.failures) + person.drop(columns=["NOEMP"], inplace=True) + without = us_esi_premiums_signal_gate(frame) + assert not without.passed + assert any("assignment provenance incomplete" in f for f in without.failures) + anchor = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not anchor.passed + assert any("assignment provenance incomplete" in f for f in anchor.failures) + + +# --- the release anchor gate -------------------------------------------------------- + + +class TestAnchorGate: + def _export(self, *, weight_scale: float = 1.0) -> Frame: + """A calibrated export: the staged people at rescaled weights.""" + + frame = _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) + return _reweighted(frame, frame.weights_for("household").values * weight_scale) + + @pytest.mark.parametrize("scale", [0.951, 1.0, 1.049]) + def test_passes_inside_the_stated_tolerance(self, scale) -> None: + gate = us_esi_premiums_anchor_gate( + self._export(weight_scale=scale), time_period=TIME_PERIOD + ) + assert gate.passed, gate.failures + assert gate.details["relative_error"] == pytest.approx(scale - 1.0) + assert gate.details["anchor_value"] == ANCHOR + assert gate.details["relative_tolerance"] == esi.ANCHOR_RELATIVE_TOLERANCE + + @pytest.mark.parametrize("scale", [0.94, 1.06]) + def test_fails_outside_the_stated_tolerance(self, scale) -> None: + gate = us_esi_premiums_anchor_gate( + self._export(weight_scale=scale), time_period=TIME_PERIOD + ) + assert not gate.passed + assert any("from BEA NIPA 7.8 line 17" in f for f in gate.failures) + + def test_the_verdict_is_over_every_policyholder_not_the_column(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) + details = gate.details + assert details["anchor_universe_employer_total"] == pytest.approx(ANCHOR) + assert details["employer_premium_total"] < ANCHOR + assert details["employer_premium_total"] + details[ + "other_policyholder_employer_total" + ] == pytest.approx(ANCHOR) + assert details["employer_premium_total"] == pytest.approx( + ANCHOR * details["employed_share_of_anchor_universe"] + ) + + def test_reweighting_workers_against_other_policyholders_is_red(self) -> None: + # Calibration that keeps the column's total but piles weight onto + # retiree policyholders moves the anchor-universe total and the split. + frame = _run(_realistic_rows(), weights=np.full(400, 100.0)) + person = frame.table("person") + retiree = (person["PEMLR"].to_numpy() == 5) & ( + person["NOW_OWNGRP"].to_numpy() == 1 + ) + weights = np.where(retiree, 500.0, 100.0) + gate = us_esi_premiums_anchor_gate( + _reweighted(frame, weights), time_period=TIME_PERIOD + ) + assert gate.details["employer_premium_total"] == pytest.approx( + _column_total(frame) + ) + assert not gate.passed + assert any("from BEA NIPA 7.8 line 17" in f for f in gate.failures) + assert any("employed policyholders carry" in f for f in gate.failures) + + def test_the_cross_checks_are_recorded_not_gated(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) + assert gate.passed, gate.failures + # NHE's estimate of the same concept is 7.2% above BEA's for 2024, so + # a release scaled to NHE would fail this gate. + assert gate.details["cross_check_ratio"] == pytest.approx(ANCHOR / NHE) + assert NHE / ANCHOR - 1 > esi.ANCHOR_RELATIVE_TOLERANCE + active = gate.details["private_active_cross_check"] + assert active["meps_ic_private_active_employer_total"] == pytest.approx( + 668.6e9, rel=5e-4 + ) + assert active["ratio"] == pytest.approx( + active["employer_premium_private_sector"] / 668.6e9, rel=5e-4 + ) + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_an_absent_column_is_red(self, column) -> None: + frame = self._export() + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not gate.passed + assert gate.failures == (f"person column missing: {column}.",) + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_zero_mass_column_is_red(self, column) -> None: + frame = self._export() + frame.table("person")[column] = 0.0 + gate = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not gate.passed + assert f"{column}: zero weighted mass." in gate.failures + + @pytest.mark.parametrize( + "column", + [c for c in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS if c != "state_fips"], + ) + def test_an_export_without_a_raw_column_is_red(self, column) -> None: + # The anchor counts every policyholder, so the gate cannot grade a + # frame that no longer says who they are. + frame = self._export() + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not gate.passed + assert any("assignment provenance incomplete" in f for f in gate.failures) + assert "anchor_universe_employer_total" not in gate.details + + def test_a_frame_without_the_household_state_fails_both_gates(self) -> None: + # State sits on the household table; without it no cell can be read. + frame = self._export() + frame.table("household").drop(columns=["state_fips"], inplace=True) + for gate in ( + us_esi_premiums_signal_gate(frame), + us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD), + ): + assert not gate.passed + assert any( + "assignment provenance incomplete" in f and "state_fips" in f + for f in gate.failures + ) + + @pytest.mark.parametrize("year", [2022, 2031]) + def test_a_year_without_an_anchor_is_red(self, year) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=year) + assert not gate.passed + assert any( + f"no BEA NIPA 7.8 line 17 anchor for {year}" in f for f in gate.failures + ) + + def test_details_serialize_for_the_release_evidence_file(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) + payload = json.loads(json.dumps(dict(gate.details))) + assert payload["anchor"]["sha256"] == esi.EMPLOYER_PREMIUM_ANCHOR["sha256"] + assert payload["anchor"]["values"]["2024"] == ANCHOR + assert payload["cross_check"]["values"]["2024"] == NHE + + +class TestOperatorBoundary: + def test_a_raw_source_frame_that_carries_the_outputs_is_refused(self) -> None: + from microcosm.build.us_runtime.operator_boundary import ( + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + assert_operator_free_source_frame, + ) + + assert PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["esi_premiums"] == { + "person": frozenset(US_ESI_PREMIUMS_OUTPUT_COLUMNS) + } + assert_operator_free_source_frame(_frame(_realistic_rows(40)), label="raw") + with pytest.raises(ValueError, match="esi_premiums:person="): + assert_operator_free_source_frame(_run(_realistic_rows(40)), label="raw") + + +class TestAnchorFeedCheck: + @staticmethod + def _facts(values: dict[int, object]) -> list[dict]: + return [ + { + "layout": { + "record_set_id": f"bea_nipa.cy{year}.private_group_health_insurance" + }, + "value": value, + } + for year, value in values.items() + ] + [{"layout": {"record_set_id": "bea_nipa.cy2024.total_wages_salaries"}}] + + def test_the_banked_series_matches_the_anchor(self) -> None: + values = { + int(year): value + for year, value in esi.EMPLOYER_PREMIUM_ANCHOR["values"].items() + } + assert values == { + 2023: 923_195_000_000, + 2024: 977_034_000_000, + 2025: 1_027_929_000_000, + } + assert esi.us_esi_premiums_anchor_feed_mismatches(self._facts(values)) == [] + + def test_the_committed_feed_pin_banks_the_anchor(self) -> None: + """The pinned feed's three added facts are the stage's anchor values. + + The feed itself is too large for the repository, so this reads the + committed row-by-row report of the re-pin. Changing the anchor without + re-pinning the feed, or the reverse, fails here and not only at a + release preflight. + """ + + report = json.loads( + ( + _REPOSITORY_ROOT / "experiments/us-esi-454/feed-repin/value_diff.json" + ).read_text() + ) + pin = json.loads( + files("microcosm.build.us").joinpath("chronicle_feed.json").read_text() + ) + assert report["new"]["sha256"] == pin["facts_sha256"] + assert report["new"]["rows"] == pin["fact_row_count"] + assert report["previous_rows_missing"] == 0 + assert report["previous_rows_changed"] == 0 + added = report["rows_added"] + assert len(added) == 3 + assert {str(row["period"]["value"]): row["value"] for row in added} == dict( + esi.EMPLOYER_PREMIUM_ANCHOR["values"] + ) + template = esi.EMPLOYER_PREMIUM_ANCHOR["feed_record_set"] + assert {row["record_set_id"] for row in added} == { + template.format(year=year) for year in esi.EMPLOYER_PREMIUM_ANCHOR["values"] + } + assert {row["source_sha256"] for row in added} == { + esi.EMPLOYER_PREMIUM_ANCHOR["sha256"] + } + # The same rows, as facts, satisfy the release tool's check. + facts = [ + {"layout": {"record_set_id": row["record_set_id"]}, "value": row["value"]} + for row in added + ] + assert esi.us_esi_premiums_anchor_feed_mismatches(facts) == [] + + def test_a_missing_repeated_or_different_fact_is_named(self) -> None: + facts = self._facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + facts += self._facts({2023: 923_195_000_000})[:1] + problems = esi.us_esi_premiums_anchor_feed_mismatches(facts) + assert len(problems) == 3 + assert any("cy2023" in p and "2 facts" in p for p in problems) + assert any("cy2024" in p and "1002920000000" in p for p in problems) + assert any("cy2025" in p and "no fact" in p for p in problems) + + +class TestUnassignedRefusal: + def test_a_staged_frame_and_a_frame_without_the_columns_pass(self) -> None: + refuse_unassigned_us_esi_premiums(_run(_realistic_rows(60)), consumer="t") + refuse_unassigned_us_esi_premiums(_frame(_realistic_rows(60)), consumer="t") + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_null_on_any_row_is_refused(self, column) -> None: + # ACS-spine rows pooled beside an ASEC donor carry no assignment. + frame = _run(_realistic_rows(60)) + frame.table("person").loc[[3, 4], column] = np.nan + with pytest.raises(SourceRuntimeError, match="never filled") as error: + refuse_unassigned_us_esi_premiums(frame, consumer="ACS local release") + assert "ACS local release" in str(error.value) + assert f"'{column}': 2" in str(error.value) + + +# --- properties and the differential reference ------------------------------------ + + +@st.composite +def _person_rows(draw) -> dict: + owner = draw(st.sampled_from([0, 1, 1, 2])) + if owner == 1: + tier = draw(st.sampled_from([1, 2, 3])) + coverage = { + "NOW_HIPAID": draw(st.sampled_from([1, 2, 2, 3])), + "NOW_GRPFTYP2": tier, + "NOW_GRPFTYP": 1 if tier in (1, 2) else 2, + } + else: + coverage = {"NOW_HIPAID": 0, "NOW_GRPFTYP2": 0, "NOW_GRPFTYP": 0} + return { + "NOW_OWNGRP": owner, + **coverage, + "PEMLR": draw(st.sampled_from([0, 1, 1, 1, 2, 3, 4, 5, 6, 7])), + "NOEMP": draw(st.integers(0, 6)), + "PEIO1COW": draw(st.sampled_from([0, 1, 2, 3, 4, 4, 4, 5, 6, 7, 8])), + "state_fips": draw(st.sampled_from(_STATES)), + } + + +@st.composite +def _populations(draw): + rows = draw(st.lists(_person_rows(), min_size=1, max_size=40)) + # At least one person with employer-paid mass, so the frame can be scaled. + rows.append( + { + "NOW_HIPAID": draw(st.sampled_from([1, 2])), + "state_fips": draw(st.sampled_from(_STATES)), + "PEIO1COW": draw(st.sampled_from([1, 2, 3, 4, 5, 6])), + } + ) + weights = draw( + st.lists( + st.floats(0.25, 9_000.0, allow_nan=False), + min_size=len(rows), + max_size=len(rows), + ) + ) + return rows, weights + + +def _reference(rows: list[dict], weights: list[float]) -> list[float]: + """A scalar restatement of the declared rules, one person at a time.""" + + tiers = {1: "family", 2: "employee_plus_one", 3: "single"} + raw: list[float] = [] + in_column: list[bool] = [] + for partial in rows: + row = {**_BASE_ROW, **partial} + holder = row["NOW_OWNGRP"] == 1 + in_universe = ( + holder and row["PEMLR"] in (1, 2) and row["PEIO1COW"] in (1, 2, 3, 4, 5, 6) + ) + size = ( + "total" + if row["NOEMP"] == 0 + else "lt50" + if row["NOEMP"] in (1, 2) + else "50plus" + ) + share = 0.0 + if holder and row["NOW_HIPAID"] in (1, 2): + tier = tiers[row["NOW_GRPFTYP2"]] + state = f"{row['state_fips']:02d}" + government = row["PEIO1COW"] in (1, 2, 3) + # No employer class: the State's all-sizes private cell. + cell_size = size if row["PEIO1COW"] in (4, 5, 6) else "total" + cell = {} + for measure in ("premium", "employee_contribution"): + private = _CELLS["private_state_2025"][tier][measure]["rows"] + if government: + division = _CELLS["state_census_division"][state] + column = "state" if row["PEIO1COW"] == 2 else "all_state_and_local" + value = ( + _CELLS["public_division_2024"][tier][measure]["rows"][division][ + column + ] + * private["US"]["total"] + / _CELLS["private_national_2024"][tier][measure]["total"] + ) + else: + value = private[state][cell_size] + if value is None: + value = ( + private["US"][cell_size] + * private[state]["total"] + / private["US"]["total"] + ) + cell[measure] = value + unpaid = ( + _CELLS["no_contribution_share_2025"][tier][ + "50plus" if government else cell_size + ] + / 100.0 + ) + share = ( + cell["premium"] + if row["NOW_HIPAID"] == 1 + else max( + cell["premium"] - cell["employee_contribution"] / (1.0 - unpaid), + 0.0, + ) + ) + raw.append(share) + in_column.append(in_universe) + # One factor over every policyholder; only the column's people keep it. + total = sum(weight * share for weight, share in zip(weights, raw, strict=True)) + return [ + share * ANCHOR / total if keep else 0.0 + for share, keep in zip(raw, in_column, strict=True) + ] + + +_PROPERTY_SETTINGS = settings( + max_examples=120, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +@_PROPERTY_SETTINGS +@given(_populations(), st.integers(0, 2**31 - 1)) +def test_stage_agrees_with_the_scalar_reference(population, seed) -> None: + rows, weights = population + result = _run(rows, weights=weights, seed=seed) + np.testing.assert_allclose( + _values(result, EMPLOYER), _reference(rows, weights), rtol=1e-11 + ) + + +@_PROPERTY_SETTINGS +@given(_populations(), st.integers(0, 2**31 - 1)) +def test_conservation_bounds_and_structural_zeros(population, seed) -> None: + rows, weights = population + result = _run(rows, weights=weights, seed=seed) + person = result.table("person") + employer = _values(result, EMPLOYER) + summary = us_esi_premiums_summary(result) + assert summary["anchor_universe_employer_total"] == pytest.approx(ANCHOR, rel=1e-11) + assert _weights(result) @ employer == pytest.approx( + ANCHOR * summary["employed_share_of_anchor_universe"], rel=1e-11 + ) + assert _weights(result) @ employer <= ANCHOR * (1 + 1e-11) + assert np.isfinite(employer).all() + assert (employer >= 0).all() + universe = ( + person["NOW_OWNGRP"].eq(1) + & person["PEMLR"].isin([1, 2]) + & person["PEIO1COW"].isin([1, 2, 3, 4, 5, 6]) + ).to_numpy() + assert (employer[~universe] == 0).all() + assert (employer[universe & person["NOW_HIPAID"].eq(3).to_numpy()] == 0).all() + assert (employer[universe & person["NOW_HIPAID"].isin([1, 2]).to_numpy()] > 0).all() + for key in ( + "nonfinite_rows", + "negative_rows", + "clone_disagreement_source_persons", + "employer_premium_outside_universe_rows", + "employer_premium_where_employer_pays_none_rows", + "employer_premium_without_raw_share_rows", + ): + assert summary[key] == 0, key + assert summary["scale_factor_spread"] <= 1e-9 * summary["scale_factor"] + + +@_PROPERTY_SETTINGS +@given(_populations(), st.randoms(use_true_random=False)) +def test_row_order_does_not_change_anyone(population, random) -> None: + rows, weights = population + keyed = [ + {**row, "person_id": index + 1, "source_household_id": index + 1} + for index, row in enumerate(rows) + ] + order = list(range(len(keyed))) + random.shuffle(order) + straight = _run(keyed, weights=weights, seed=3).table("person") + shuffled = _run( + [keyed[index] for index in order], + weights=[weights[index] for index in order], + seed=3, + ).table("person") + straight = straight.set_index("person_id").sort_index() + shuffled = shuffled.set_index("person_id").sort_index() + np.testing.assert_allclose( + shuffled[EMPLOYER].to_numpy(), straight[EMPLOYER].to_numpy(), rtol=1e-11 + ) + + +@_PROPERTY_SETTINGS +@given(_populations(), st.floats(1e-3, 1e3, allow_nan=False)) +def test_weights_are_homogeneous_of_degree_minus_one(population, factor) -> None: + rows, weights = population + base = _run(rows, weights=weights) + scaled = _run(rows, weights=[weight * factor for weight in weights]) + np.testing.assert_allclose( + _values(scaled, EMPLOYER) * factor, _values(base, EMPLOYER), rtol=1e-10 + ) + assert _column_total(scaled) == pytest.approx(_column_total(base), rel=1e-10) + assert us_esi_premiums_summary(scaled)[ + "anchor_universe_employer_total" + ] == pytest.approx(ANCHOR, rel=1e-11) + + +@_PROPERTY_SETTINGS +@given(_populations(), st.integers(0, 2**31 - 1)) +def test_staging_is_deterministic_restageable_and_clone_safe(population, seed) -> None: + rows, weights = population + once = _run(rows, weights=weights, seed=seed) + # The assignment draws nothing, so another seed changes no one. + again = _run(rows, weights=weights, seed=seed + 1) + # Staging a staged frame again from its raw columns changes no one: the + # stage reads only the raw ASEC columns, never its own outputs. + stripped = _reweighted(once, once.weights_for("household").values) + stripped.table("person").drop( + columns=list(US_ESI_PREMIUMS_OUTPUT_COLUMNS), inplace=True + ) + restaged = with_us_esi_premium_inputs(stripped, seed=seed, time_period=TIME_PERIOD) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + assert _values(again, column).tolist() == _values(once, column).tolist() + assert _values(restaged, column).tolist() == _values(once, column).tolist() + cloned = _cloned(once) + assert us_esi_premiums_summary(cloned)["clone_disagreement_source_persons"] == 0 + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + assert _weights(cloned) @ _values(cloned, column) == pytest.approx( + _weights(once) @ _values(once, column), rel=1e-11, abs=1e-6 + ) + assert us_esi_premiums_summary(cloned)[ + "anchor_universe_employer_total" + ] == pytest.approx(ANCHOR, rel=1e-11) + + +@_PROPERTY_SETTINGS +@given(_populations()) +def test_pricing_other_policyholders_never_raises_a_workers_premium(population) -> None: + rows, weights = population + priced = _run(rows, weights=weights) + person = priced.table("person") + universe = ( + person["NOW_OWNGRP"].eq(1) + & person["PEMLR"].isin([1, 2]) + & person["PEIO1COW"].isin([1, 2, 3, 4, 5, 6]) + ).to_numpy() + # The same frame with every policyholder outside the column uncovered: + # nothing is left to price but the column, so it carries the whole anchor. + workers_only = _run( + [ + row if keep else {**row, **_NO_COVERAGE} + for row, keep in zip(rows, universe, strict=True) + ], + weights=weights, + ) + assert _column_total(workers_only) == pytest.approx(ANCHOR, rel=1e-11) + assert ( + _values(priced, EMPLOYER) <= _values(workers_only, EMPLOYER) * (1 + 1e-11) + ).all() + assert _column_total(priced) <= ANCHOR * (1 + 1e-11) + + +@_PROPERTY_SETTINGS +@given( + st.sampled_from([1, 2, 3]), + st.sampled_from([1, 2, 3, 4, 5, 6]), + st.sampled_from([4, 5, 6]), + st.sampled_from(_STATES), + st.floats(0.5, 500.0, allow_nan=False), +) +def test_the_published_mix_reconciles_sampled_private_cells_through_the_stage( + tier_code, noemp, sector, state, mass +) -> None: + tier = esi._TIER_BY_CODE[tier_code] + share = _no_contribution(tier, esi._SIZE_BY_NOEMP[noemp]) + cell = { + "NOW_GRPFTYP2": tier_code, + "NOW_GRPFTYP": 1 if tier_code in (1, 2) else 2, + "NOEMP": noemp, + "PEIO1COW": sector, + "state_fips": state, + } + result = _run( + [{**cell, "NOW_HIPAID": 1}, {**cell, "NOW_HIPAID": 2}], + weights=[mass * share, mass * (1 - share)], + ) + summary = us_esi_premiums_summary(result) + assert summary["raw_over_published_employer_mean"] == pytest.approx(1.0) + + +# --- the registers the stage moves ------------------------------------------------- + + +class TestRegisters: + def test_the_input_is_a_required_release_column_with_no_exclusion(self) -> None: + required = us_release_input_coverage_required_columns() + assert EMPLOYER in required + assert EMPLOYER not in set(us_release_input_coverage_reviewed_exclusions()) + # Not produced, so not required: the pre-tax employee premium waits + # for the engine's disjoint premium contract (policyengine-us#10046). + assert "pre_tax_health_insurance_premiums" not in required + + def test_the_parity_gap_register_no_longer_exempts_the_employer_premium( + self, + ) -> None: + payload = json.loads( + files("microcosm.build.us") + .joinpath("ecps_parity_known_gaps.json") + .read_text() + ) + assert EMPLOYER not in payload["known_gaps"] + + def test_a_shipped_reform_probe_binds_through_the_input(self) -> None: + probes = {probe.id: probe for probe in us_release_reform_coverage_probes()} + employer = probes["employer_sponsored_insurance_premium_neutralization"] + assert employer.neutralized_variable == EMPLOYER + assert employer.budget_measure == "cbo_household_market_income" + assert employer.effect_direction == "baseline_minus_reform" + assert employer.expected_sign == "positive" + assert 0 < employer.min_abs_effect < ANCHOR + assert "pre_tax_health_insurance_premium_neutralization" not in probes + + def test_the_anchor_and_cross_check_families_are_gated_not_compiled( + self, + ) -> None: + manifest = json.loads( + files("microcosm.build.us") + .joinpath("target_parity_manifest.json") + .read_text() + ) + families = manifest["families"] + anchor = families["bea_nipa.private_group_health_insurance"] + assert anchor["status"] == "reviewed_exclusion" + assert anchor["classification"] == "deferred" + assert "scale anchor" in anchor["reason"] + assert "B4923C" in anchor["reason"] + assert "active and retired" in anchor["reason"] + assert "us_esi_premiums_anchor_gate" in anchor["reason"] + assert "us_esi_premiums_anchor_feed_mismatches" in anchor["reason"] + assert "d1264" in anchor["fence"]["verdict_basis"] + nhe = families["cms_nhe.esi_employer_contribution_premiums"] + assert nhe["classification"] == "deferred" + assert "retirees" in nhe["reason"] and "d1317" in nhe["reason"] + private = families["cms_nhe.esi_private_employer_contribution_premiums"] + assert private["classification"] == "deferred" + assert "no gate reads this fact" in private["fence"]["verdict_basis"] + # Each family states its own verdict basis. + assert ( + len({entry["fence"]["verdict_basis"] for entry in (anchor, nhe, private)}) + == 3 + ) + # The BEA facts are in the pinned feed now, so the family is no longer + # source-absent. + assert not any( + entry.get("classification") == "source_absent" + and family.startswith("bea_nipa.private_group_health") + for family, entry in families.items() + ) + + @pytest.mark.parametrize( + ("pool", "scale", "holders", "column", "mean", "share"), + [ + # The default pool (docs/us-asec-source-pins.md) and the historical + # Build J/N/P pool; figures quoted in the docs and the registers. + ("2023_2025", 0.9708, 73.97e6, 863.85e9, 11_679, 0.8842), + ("2022_2024", 0.9653, 74.06e6, 862.16e9, 11_642, 0.8824), + ], + ) + def test_the_receipts_reproduce_the_documented_measurements( + self, pool, scale, holders, column, mean, share + ) -> None: + receipt = json.loads( + ( + _REPOSITORY_ROOT + / f"experiments/us-esi-454/receipts/stage_on_pool_{pool}.json" + ).read_text() + ) + summary = receipt["summary"] + years = [int(year) for year in pool.split("_")] + assert receipt["pooled_income_years"] == list(range(years[0], years[1] + 1)) + assert receipt["signal_gate"] == {"passed": True, "failures": []} + assert receipt["anchor_gate"]["passed"] is True + clone = receipt["puf_support_clone"] + assert clone["signal_gate"]["passed"] and clone["anchor_gate"]["passed"] + for totals in (summary, clone["anchor_gate"]): + assert totals["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + assert totals["employer_premium_total"] == pytest.approx(column, rel=1e-4) + assert summary["cells_sha256"] == esi._CELLS_SHA256 + assert summary["source_columns_missing"] == [] + assert summary["scale_factor"] == pytest.approx(scale, abs=5e-5) + assert summary["employed_share_of_anchor_universe"] == pytest.approx( + share, abs=5e-5 + ) + assert summary["employer_premium_positive_persons"] == pytest.approx( + holders, rel=1e-3 + ) + assert summary["employer_premium_mean_per_positive_person"] == pytest.approx( + mean, abs=1 + ) + details = receipt["anchor_gate"]["details"] + assert details["cross_check_ratio"] == pytest.approx(ANCHOR / NHE) + active = details["private_active_cross_check"] + assert active["meps_ic_private_active_employer_total"] == pytest.approx( + 668.6e9, rel=5e-4 + ) + # The private part of the column against MEPS-IC's own active total. + assert 0.99 < active["ratio"] < 1.0 + sensitivity = receipt["other_policyholder_sensitivity"] + assert sensitivity["as_built_other_priced_as_active"][ + "employer_premium_total" + ] == pytest.approx(summary["employer_premium_total"]) + assert sensitivity["other_not_priced_all_anchor_on_employed"][ + "employer_premium_total" + ] == pytest.approx(ANCHOR) + assert ( + summary["employer_premium_total"] + < sensitivity["other_65_plus_priced_at_half"]["employer_premium_total"] + < ANCHOR + ) + restored = ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2"] + for source in receipt["pool_census_person_columns"]: + assert source["columns_added"][-6:] == [*restored, "PEMLR", "NOEMP"] + + +# --- the cell-table extraction tool --------------------------------------------------- + + +def _load_cells_tool(): + path = _REPOSITORY_ROOT / "tools" / "build_us_meps_ic_esi_cells.py" + spec = importlib.util.spec_from_file_location("build_us_meps_ic_esi_cells", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class TestCellsTool: + def test_committed_cells_carry_the_tools_pins(self) -> None: + tool = _load_cells_tool() + assert {pin.key: pin.sha256 for pin in tool.PDFS} == { + key: source["sha256"] for key, source in _CELLS["sources"].items() + } + assert tool.render(dict(_CELLS)) == tool.OUTPUT_PATH.read_text() + + def test_row_parser_reads_suppression_and_reliability_marks(self) -> None: + tool = _load_cells_tool() + block = [ + "Table II.D.2 Average total employee contribution: United States, 2025", + "United States 7,314 6,113 7,721 9,242 7,310 7,050 7,760 7,246", + "Alaska 6,993 -- 5,811 * 8,001 7,406 6,750 -- 7,184", + "West Virginia 6,100 5,000 5,500 6,000 6,300 6,200 5,400 6,250", + "Virginia 7,000 6,500 6,800 7,100 7,200 6,900 6,700 7,050", + ] + values, unreliable = tool._row_values(block, "Alaska", 8, "II.D.2") + assert values == [6993, None, 5811, 8001, 7406, 6750, None, 7184] + assert unreliable == [False, False, True, False, False, False, False, False] + virginia, _ = tool._row_values(block, "Virginia", 8, "II.D.2") + assert virginia[0] == 7000 + + def test_a_flagged_or_suppressed_national_row_is_refused(self) -> None: + tool = _load_cells_tool() + pin = next(pin for pin in tool.PDFS if pin.key == "private_state_2025") + head = "Table II.C.4.a Percent of enrollees: United States, 2025" + good = "United States 13.0% 52.7% 32.6% 24.4% 13.4% 4.8% 35.5% 8.7%" + row = tool._national_row([head, good], pin, "II.C.4.a") + assert (row["total"], row["lt50"], row["50plus"]) == (13.0, 35.5, 8.7) + # A flag on a column the stage does not read is tolerated. + tolerated = good.replace("4.8%", "4.8% *") + assert tool._national_row([head, tolerated], pin, "II.C.4.a")["lt50"] == 35.5 + for bad in (good.replace("35.5%", "35.5% *"), good.replace("8.7%", "--")): + with pytest.raises(tool.ExtractionError, match="suppressed or flagged"): + tool._national_row([head, bad], pin, "II.C.4.a") + + def test_row_parser_refuses_a_missing_duplicated_or_short_row(self) -> None: + tool = _load_cells_tool() + row = "Ohio 1 2 3" + with pytest.raises(tool.ExtractionError, match="matched 0 rows"): + tool._row_values([row], "Iowa", 3, "X") + with pytest.raises(tool.ExtractionError, match="matched 2 rows"): + tool._row_values([row, row], "Ohio", 3, "X") + with pytest.raises(tool.ExtractionError, match="has 3 values, not 8"): + tool._row_values([row], "Ohio", 8, "X") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 9adfe409b7..27c68451e6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -6097,6 +6097,13 @@ def refuse_materialization(*args, **kwargs): assert not out.exists() inputs = payload["inputs"] assert inputs["git_dirty"] is True + # The fixture's feed check finds no difference (microcosm#454). + assert inputs["esi_anchor_feed"] == { + "enforced": True, + "matches": True, + "mismatches": [], + } + assert inputs["waivers"]["allow_unpinned_feed"] is False # The stop point's time is taken before any grading. if mode == "dry_run_refusal": assert inputs["seconds_to_stop_point"] is None @@ -6493,11 +6500,30 @@ def fake_smoke(*, simulate, period): ), } expected_gate_evidence = { + "esi_premiums": "bound", "input_coverage": "bound", "input_mass_parity": "bound", "qrf_tail_concentration": "bound", "reform_coverage_smoke": "skipped" if skipped_smoke else "bound", } + # microcosm#454: the ESI premium signal gate graded the base before the + # solve and the export after it; the anchor gate graded the export at the + # release period and its verdict is the bound esi_premiums evidence, with + # both signal verdicts recorded beside it. + assert captured["esi_premiums_signal_gate_calls"] == 2 + assert captured["esi_premiums_anchor_periods"] == [builder.PERIOD] + signal_evidence = {"evaluated": True, "passed": True, "failures": []} + assert json.loads((release_dir / "esi_premiums.json").read_text()) == { + "schema_version": 3, + "enforced": True, + "esi_premiums": { + "passed": True, + "failures": [], + "details": {"checked": True}, + }, + "anchor_feed": {"enforced": True, "matches": True, "mismatches": []}, + "signal": {"base_frame": signal_evidence, "export_frame": signal_evidence}, + } for block in (build_manifest, release_manifest["build"]): assert block["gate_evidence"] == expected_gate_evidence assert block["qrf_tail_register"] == expected_register @@ -7151,10 +7177,14 @@ def fake_written_stored_input_check(path, pre_export): ), ) # The invented feed is not the committed pin; this test is about the - # corridor contract, not the pin, which has its own tests. + # corridor contract, not the pin, which has its own tests. Nor does it + # carry the ESI anchor's B4923C facts (checked in test_us_esi_premiums). monkeypatch.setattr( builder, "_check_committed_us_ledger_feed_pin", lambda *a, **k: None ) + monkeypatch.setattr( + builder, "us_esi_premiums_anchor_feed_mismatches", lambda facts: [] + ) monkeypatch.setattr( builder, "compile_us_fiscal_target_registry", @@ -8343,6 +8373,32 @@ def fake_other_health_insurance_signal_gate(frame): "us_other_health_insurance_signal_gate", fake_other_health_insurance_signal_gate, ) + + def fake_esi_premiums_signal_gate(frame): + # Called on the base before the solve and on the export after it. + captured["esi_premiums_signal_gate_calls"] = ( + captured.get("esi_premiums_signal_gate_calls", 0) + 1 + ) + return builder.GateResult( + name="esi_premiums_signal", + passed=True, + details={"checked": True}, + ) + + def fake_esi_premiums_anchor_gate(frame, *, time_period): + captured.setdefault("esi_premiums_anchor_periods", []).append(time_period) + return builder.GateResult( + name="esi_premiums_anchor", + passed=True, + details={"checked": True}, + ) + + monkeypatch.setattr( + builder, "us_esi_premiums_signal_gate", fake_esi_premiums_signal_gate + ) + monkeypatch.setattr( + builder, "us_esi_premiums_anchor_gate", fake_esi_premiums_anchor_gate + ) monkeypatch.setattr( builder, "_with_snap_state_take_up_outputs", @@ -12574,7 +12630,7 @@ def test_build_manifests_binds_gate_evidence_and_the_qrf_tail_register( monkeypatch, tmp_path ) -> None: """Route A remediation PR-3: a certified waiver must ship with the - release. The four gate verdicts become release artifacts, and both + release. The five gate verdicts become release artifacts, and both manifests record the register the tail gate evaluated, the export-mass reference, the solve's thread geometry and the exclusion receipt.""" from microcosm.data.contract import ( @@ -12611,7 +12667,11 @@ def test_build_manifests_binds_gate_evidence_and_the_qrf_tail_register( == [] ) assert failures == [] - for filename in ("input_coverage.json", "input_mass_parity.json"): + for filename in ( + "esi_premiums.json", + "input_coverage.json", + "input_mass_parity.json", + ): (release_dir / filename).write_text('{"schema_version": 1, "enforced": true}') (release_dir / "reform_coverage_smoke.json").write_text( '{"schema_version": 1, "enforced": true}' @@ -12737,6 +12797,7 @@ def test_build_manifests_records_a_blanket_waiver_without_register_and_absent_re assert build["fiscal_target_exclusion_receipt"]["receipt_sha256"] is None # A verdict that was not written is named, never silently missing. assert build["gate_evidence"] == { + "esi_premiums": "not_evaluated", "input_coverage": "not_evaluated", "input_mass_parity": "not_evaluated", "qrf_tail_concentration": "bound", @@ -12761,13 +12822,14 @@ def test_build_manifests_without_gate_evidence_binds_none() -> None: def test_gate_evidence_status_names_skipped_and_unevaluated_gates() -> None: """A skipped smoke and a gate that crashed under earlier failures both leave no verdict; the status block tells them apart. A gate cannot be - both skipped and bound, and only the four bound gates can be skipped.""" + both skipped and bound, and only the five bound gates can be skipped.""" builder = _load_builder_module() bound = {"input_coverage": {}, "input_mass_parity": {}} assert builder._gate_evidence_status( bound, skipped_gates=("reform_coverage_smoke",) ) == { + "esi_premiums": "not_evaluated", "input_coverage": "bound", "input_mass_parity": "bound", "qrf_tail_concentration": "not_evaluated", @@ -16336,6 +16398,83 @@ def test_main_checks_the_committed_feed_pin_before_compiling_targets() -> None: assert source.count("_check_committed_us_ledger_feed_pin(") == 1 +def test_main_holds_the_esi_anchor_to_the_feed_before_compiling_targets() -> None: + """microcosm#454: the anchor-versus-feed check runs on the pinned feed.""" + builder = _load_builder_module() + source = inspect.getsource(builder._main) + checked = source.index("_check_committed_us_ledger_feed_pin(") + anchored = source.index( + "_check_us_esi_anchor_against_feed(\n ledger_artifact.facts" + ) + compiled = source.index("target_registry = compile_us_fiscal_target_registry(") + assert checked < anchored < compiled + assert source.count("_check_us_esi_anchor_against_feed(") == 1 + # The same list reaches the evidence file. + assert "_esi_anchor_feed_evidence(\n" in source + + +def _b4923c_facts(values: dict[int, int]) -> list[dict]: + return [ + { + "layout": { + "record_set_id": f"bea_nipa.cy{year}.private_group_health_insurance" + }, + "value": value, + } + for year, value in values.items() + ] + + +def test_the_esi_anchor_feed_check_passes_on_the_banked_series() -> None: + builder = _load_builder_module() + facts = _b4923c_facts( + {2023: 923_195_000_000, 2024: 977_034_000_000, 2025: 1_027_929_000_000} + ) + assert ( + builder._check_us_esi_anchor_against_feed( + facts, allow_unpinned_feed=False, allow_esi_premium_gaps=False + ) + == [] + ) + + +def test_the_esi_anchor_feed_check_refuses_a_feed_that_disagrees() -> None: + """A revised or missing B4923C fact stops a release build.""" + builder = _load_builder_module() + # BEA's pre-revision 2024 value, and no 2025 fact. + facts = _b4923c_facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + with pytest.raises(SystemExit, match="anchor differs from the Chronicle feed"): + builder._check_us_esi_anchor_against_feed( + facts, allow_unpinned_feed=False, allow_esi_premium_gaps=False + ) + + +@pytest.mark.parametrize( + ("allow_unpinned_feed", "allow_esi_premium_gaps"), + [(True, False), (False, True), (True, True)], +) +def test_a_waived_esi_anchor_feed_mismatch_stays_on_the_record( + allow_unpinned_feed, allow_esi_premium_gaps +) -> None: + """A diagnostic flag lets the build go on; the difference is recorded.""" + builder = _load_builder_module() + facts = _b4923c_facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + mismatches = builder._check_us_esi_anchor_against_feed( + facts, + allow_unpinned_feed=allow_unpinned_feed, + allow_esi_premium_gaps=allow_esi_premium_gaps, + ) + assert len(mismatches) == 2 + evidence = builder._esi_anchor_feed_evidence(mismatches, enforced=False) + assert evidence == { + "enforced": False, + "matches": False, + "mismatches": mismatches, + } + assert any("1002920000000" in item for item in evidence["mismatches"]) + assert any("cy2025" in item for item in evidence["mismatches"]) + + # SPM measurement composition refusal # --------------------------------------------------------------------------- @@ -17134,3 +17273,63 @@ def test_main_reports_actual_dry_run_outcome( if expected_status == "failed": assert str(exit_code) in run_events[0]["message"] assert run_events[0]["details"]["failure_class"] == "build_failure" + + +def test_a_dry_run_reports_a_waived_esi_anchor_feed_mismatch() -> None: + """microcosm#454: a dry run stops before esi_premiums.json, so the report + carries a B4923C difference that --allow-unpinned-feed let through.""" + + builder = _load_builder_module() + args = SimpleNamespace( + evidence_release=False, + allow_qrf_tail_concentration=False, + allow_input_mass_drift=False, + allow_ecps_parity_gaps=False, + allow_input_coverage_gaps=False, + allow_esi_premium_gaps=False, + allow_unpinned_feed=True, + ) + dry_run = builder._ReleaseDryRun( + args=args, + argv=("--allow-unpinned-feed",), + report_path=Path("report.json"), + margins=SimpleNamespace(to_dict=lambda: {}), + git_dirty=False, + started=0.0, + ) + assert dry_run.inputs()["esi_anchor_feed"] == { + "enforced": False, + "matches": True, + "mismatches": [], + } + facts = _b4923c_facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + dry_run.esi_anchor_feed_mismatches = builder._check_us_esi_anchor_against_feed( + facts, allow_unpinned_feed=True, allow_esi_premium_gaps=False + ) + inputs = dry_run.inputs() + assert inputs["waivers"]["allow_unpinned_feed"] is True + assert inputs["esi_anchor_feed"]["enforced"] is False + assert inputs["esi_anchor_feed"]["matches"] is False + assert len(inputs["esi_anchor_feed"]["mismatches"]) == 2 + + +def test_esi_premium_signal_evidence_keeps_a_waived_failure_on_the_record() -> None: + """microcosm#454: --allow-esi-premium-gaps records, it does not erase. + + The evidence file carries the base and export signal verdicts, so a build + that waived an assignment failure still says which check was red. A gate + that never ran (it crashed under earlier failures) is recorded as such. + """ + + builder = _load_builder_module() + failed = builder.GateResult( + name="esi_premiums_signal", + passed=False, + failures=("ESI premiums: 100 employer premium(s) outside employed ...",), + ) + assert builder._esi_premiums_signal_evidence(failed) == { + "evaluated": True, + "passed": False, + "failures": ["ESI premiums: 100 employer premium(s) outside employed ..."], + } + assert builder._esi_premiums_signal_evidence(None) == {"evaluated": False} diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py index ce708331ae..628538479f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py @@ -59,6 +59,7 @@ def _us_frame(**person_extra: object) -> Frame: "takes_up_wic_if_eligible": [True, False, True], "educator_expense": [0.0, 300.0, 150.0], "other_health_insurance_premiums": [1_200.0, 0.0, 600.0], + "employer_sponsored_insurance_premiums": [9_400.0, 0.0, 6_100.0], "health_insurance_premiums": [1_200.0, 0.0, 800.0], "is_self_employed": [True, False, True], "is_incapable_of_self_care": [False, True, False], @@ -908,6 +909,37 @@ def test_required_us_release_source_columns_enforces_educator_expense_signal() - assert_required_us_release_source_columns(raw_frame) +def test_required_us_release_source_columns_enforces_esi_premium_signal() -> None: + """microcosm#454: an L0/refit export refuses an absent or flat ESI column.""" + + frame = _us_frame() + for mutate, message in ( + ( + lambda people: people.drop( + columns=["employer_sponsored_insurance_premiums"] + ), + "person.employer_sponsored_insurance_premiums: missing", + ), + ( + lambda people: people.assign(employer_sponsored_insurance_premiums=0.0), + "person.employer_sponsored_insurance_premiums: not nonconstant", + ), + ): + raw_frame = Frame( + { + **{ + entity: frame.table(entity).copy() + for entity in frame.schema.entities + }, + "person": mutate(frame.table("person").copy()), + }, + frame.schema, + {"household": frame.weights_for("household")}, + ) + with pytest.raises(ValueError, match=message): + assert_required_us_release_source_columns(raw_frame) + + def test_required_us_release_source_columns_enforces_other_premium_signal() -> None: frame = _us_frame() raw_people = frame.table("person").copy() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py index bd3d3c2284..8fdc9ed67d 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py @@ -561,77 +561,45 @@ def test_legacy_two_column_surface_is_rebuilt_or_refused( ) -def test_esi_exclusion_pins_complete_hermetic_source_unavailability_evidence() -> None: +def test_esi_employer_premium_is_owned_by_its_own_stage_not_proxied_here() -> None: + """microcosm#454 closed the source-unavailability gap this test once pinned. + + The retired derivation's NOW_* fields are restored from the pinned Census + person files and the ``meps_esi_premiums`` stage reads them. This stage + still never proxies the employer premium, and neither stage may fall back + on ``has_esi`` or ``NOW_GRP``. + """ + payload = json.loads( files("microcosm.build.us").joinpath("ecps_parity_known_gaps.json").read_text() ) - entry = payload["known_gaps"]["employer_sponsored_insurance_premiums"] - evidence = entry["evidence"] - - assert entry["reason"].startswith("SOURCE UNAVAILABILITY WITH EVIDENCE:") - assert evidence["classification"] == "source_unavailability" - assert evidence["retired_derivation"] == { - "repository_owner": "PolicyEngine", - "repository_name_parts": ["policyengine-", "us-data"], - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "cps", "cps.py"], - "lines": "197-271,1575-1581", - } - assert evidence["optional_source_fields"]["lines"] == "13-55" - assert evidence["required_columns"] == [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - "PHIP_VAL", - ] - evidence_hashes = { - item["filename"]: item["sha256"] for item in evidence["hermetic_inputs"] - } - build_summary = json.loads( - (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() - ) - recorded_hashes = { - Path(item["path"]).name: item["sha256"] - for item in build_summary["base_source"]["sources"] - } - assert evidence_hashes == recorded_hashes - assert set(evidence_hashes) == { - "census_cps_2022.h5", - "census_cps_2023.h5", - "census_cps_2024.h5", - } - for item in evidence["hermetic_inputs"]: - assert item["present_columns"] == ["PHIP_VAL"] - assert item["missing_columns"] == [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - ] - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] - build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() - for year in (2022, 2023, 2024): - assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script + assert "employer_sponsored_insurance_premiums" not in payload["known_gaps"] esi_stage = US_SOURCE_MANIFEST.stage_map()["meps_esi_premiums"] - assignment = next( - operation - for operation in esi_stage.operations - if operation.kind == "assign_by_plan_type" + assert "employer_sponsored_insurance_premiums" in esi_stage.outputs + assert "employer_sponsored_insurance_premiums" not in ( + us_other_health_insurance_stage_spec().outputs ) - assert assignment.parameters["inputs"] == [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - "PHIP_VAL", - ] - assert assignment.parameters["source_unavailable_when_missing"] == [ + microdata = next( + artifact + for artifact in esi_stage.artifacts + if artifact["kind"] == "public_microdata" + ) + source_columns = set(microdata["source_columns"]) + assert { "NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", - ] - forbidden_proxies = {"has_esi", "is_esi_dependent", "tax_unit_size", "state_fips"} - assert forbidden_proxies.isdisjoint(assignment.parameters["inputs"]) + "NOW_GRPFTYP2", + } <= source_columns + # The employer premium never reads the reported premium: PHIP_VAL stays + # this stage's source alone. + assert "PHIP_VAL" not in source_columns + forbidden_proxies = {"has_esi", "NOW_GRP", "is_esi_dependent", "tax_unit_size"} + assert forbidden_proxies.isdisjoint(source_columns) + assert not any( + operation.kind == "assign_by_plan_type" for operation in esi_stage.operations + ) def test_shipped_se_health_premium_neutralization_probe() -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py index 6bbccdc296..51fa1a802f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py @@ -1275,6 +1275,10 @@ class TestBaseStageSourceClosure: "source_person_id", "source_row_id", "person_source_id", + # The household State (GESTFIPS) the pool attaches to every person + # and unit construction moves to the household table; the + # meps_esi_premiums stage joins it back by person_household_id. + "state_fips", } ) @@ -1283,6 +1287,14 @@ class TestBaseStageSourceClosure: #: 2022; ED_VAL and PAW_TYP for every pooled year). SIDECAR_RESTORED_COLUMNS = frozenset({"LKWEEKS", "ED_VAL", "PAW_TYP"}) + #: Census person columns asec_census_person_columns restores into every + #: pooled vintage, by exact PERIDNUM identity, that NO frozen input carries + #: (microcosm #454): the meps_esi_premiums stage's coverage, employment + #: and employer-size fields. + CENSUS_PERSON_RESTORED_COLUMNS = frozenset( + {"NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2", "PEMLR", "NOEMP"} + ) + #: Release-time stage constants whose inputs are produced inside the #: fiscal-refresh release tool, not the base builder (org_wages consumes #: hours/weeks columns the release derives upstream of it). @@ -1323,6 +1335,7 @@ def test_every_base_stage_requirement_has_a_provider(self) -> None: self.CENSUS_CPS_PERSON_COLUMNS | self.POOL_CONSTRUCTED_COLUMNS | self.SIDECAR_RESTORED_COLUMNS + | self.CENSUS_PERSON_RESTORED_COLUMNS | produced ) unsourced: dict[str, list[str]] = {} @@ -1338,6 +1351,15 @@ def test_every_base_stage_requirement_has_a_provider(self) -> None: ] if missing: unsourced[name] = missing + from microcosm.build.us_runtime.asec_census_person_columns import ( + ASEC_CENSUS_PERSON_COLUMN_NAMES, + ) + + # The restoration really does carry every column declared restored. + assert self.CENSUS_PERSON_RESTORED_COLUMNS <= set( + ASEC_CENSUS_PERSON_COLUMN_NAMES + ) + assert not self.CENSUS_PERSON_RESTORED_COLUMNS & self.CENSUS_CPS_PERSON_COLUMNS assert unsourced == {}, ( "Stage source columns with no declared provider (wire an ingestion " f"carry, a pinned sidecar, or an upstream stage output): {unsourced}" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py index fd4477bb4d..22ab466265 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py @@ -75,7 +75,7 @@ def test_batched_engines_partition_release_and_order_their_work( validation = consumers["reform_validation"].record() demographics = consumers["demographics"].record() probes = len(us_release_reform_coverage_probes()) - assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 41 + assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 42 assert validation["reform_systems"] == validation["reform_passes"] > 0 assert demographics["reform_systems"] == demographics["reform_passes"] == 0 assert all(engine.reform is None for engine in constructions) @@ -1540,7 +1540,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( ) -> None: sweep = _load_sweep_module() plans, requests, counts = sweep._record_requests(builder) - assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 41 + assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 42 specs = reform_validation_module.load_default_reform_specs(period=builder.PERIOD) assert counts["validation_specs"] == sum(not spec.in_sample for spec in specs) == 52 assert ( @@ -1549,7 +1549,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( == 12 ) assert counts["baseline_keys"] == { - "reform_coverage_smoke": 18, + "reform_coverage_smoke": 19, "reform_validation": 75, "demographics": 1, } @@ -1584,7 +1584,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( assert ( len(requests) == counts["smoke_probes"] + counts["validation_reform_passes"] - == 107 + == 108 ) frame, log = _nested_frame(), _EngineLog() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py index e6944adaae..82ebe5574e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py @@ -1000,6 +1000,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", + "with_us_esi_premium_inputs", "with_us_immigration_inputs", ) for name in identity_transforms: @@ -1013,6 +1014,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: "us_eligibility_inputs_signal_gate", "us_pregnancy_signal_gate", "us_wic_claim_signal_gate", + "us_esi_premiums_signal_gate", ): monkeypatch.setattr(builder, name, lambda _frame: passing_gate) @@ -1180,6 +1182,7 @@ def test_source_and_preclone_stages_round_trip_design_weight_kind( "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", + "with_us_esi_premium_inputs", "with_us_immigration_inputs", ) for name in identity_transforms: @@ -1193,6 +1196,7 @@ def test_source_and_preclone_stages_round_trip_design_weight_kind( "us_eligibility_inputs_signal_gate", "us_pregnancy_signal_gate", "us_wic_claim_signal_gate", + "us_esi_premiums_signal_gate", ): monkeypatch.setattr(builder, name, lambda _frame: passing_gate) monkeypatch.setattr(builder, "_sha256", lambda _path: "fixture-sha256") @@ -2322,6 +2326,7 @@ def fake_housing_inputs_signal_gate(frame): for gate_name in ( "us_eligibility_inputs_signal_gate", "us_pregnancy_signal_gate", + "us_esi_premiums_signal_gate", ): monkeypatch.setattr( builder, @@ -2330,6 +2335,13 @@ def fake_housing_inputs_signal_gate(frame): "Gate", (), {"passed": True, "failures": (), "details": {}} )(), ) + esi_premium_calls: list[object] = [] + + def fake_esi_premium_inputs(frame, *, seed, time_period): + esi_premium_calls.append((frame, seed, time_period)) + return frame + + monkeypatch.setattr(builder, "with_us_esi_premium_inputs", fake_esi_premium_inputs) monkeypatch.setattr( builder, "us_wic_claim_signal_gate", @@ -2862,6 +2874,10 @@ def fake_retirement_distributions_signal_gate(frame): if failing_gate == "retirement_distributions" else [] ) + # microcosm#454: the ESI premium stage runs once, on the ASEC frame right + # after the direct retirement-distribution pass and before the clone; the + # clone carries its columns unchanged, so it never re-runs on the PUF half. + assert esi_premium_calls == [("retirement-distributions-direct", 7, 2024)] def test_resume_retirement_stage_forces_puf_imputation() -> None: @@ -2890,6 +2906,29 @@ def test_main_summary_records_retirement_distribution_gate() -> None: assert '"details": dict(retirement_distributions_gate.details)' in source +def test_main_summary_records_esi_premium_gates() -> None: + """microcosm#454: the base summary carries the stage lineage twice, on the + ASEC frame and again on the support-cloned frame, in both build paths.""" + builder = _load_support_builder_module() + source = Path(builder.__file__).read_text(encoding="utf-8") + + assert '"esi_premiums_signal": {' in source + assert '"passed": esi_premiums_gate.passed' in source + assert '"details": dict(esi_premiums_gate.details)' in source + assert '"esi_premiums_post_clone_signal": {' in source + assert '"details": dict(esi_premiums_post_clone_gate.details)' in source + assert 'signals["esi_premiums_signal"] = _checked_gate_payload(' in source + assert ( + 'signals["esi_premiums_post_clone_signal"] = _checked_gate_payload(' in source + ) + for name in ("esi_premiums_signal", "esi_premiums_post_clone_signal"): + assert f' "{name}",' in source # the staged summary requires it + boundaries = dict(builder.STAGE_BOUNDARIES)["pre_clone_enrichment"] + assert boundaries.index("with_us_esi_premium_inputs") == ( + boundaries.index("with_us_retirement_distribution_inputs") + 1 + ) + + def test_main_summary_records_weeks_unemployed_gate() -> None: builder = _load_support_builder_module() source = Path(builder.__file__).read_text(encoding="utf-8") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 284149bdf1..9df5868f13 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -266,6 +266,11 @@ "education_assistance_source.py", "eligibility_inputs.py", "engine_lifecycle.py", + # MEPS-IC employer-premium input (#454): a base-builder pre-clone + # stage keyed on restored ASEC coverage fields, + # not yet a multispine pool operator. Reads no spine or support-channel + # column and remains subject to the all-runtime source-identity scan. + "esi_premiums.py", "exact_k_ladder.py", # Calibration/selection seam; no source treatment. "fiscal_targets.py", "geography_ladder.py", @@ -3486,15 +3491,17 @@ def test_pool_build_tool_import_graph_is_source_spine_blind() -> None: for tool in _SPINE_BLIND_BUILD_TOOLS: runtime_graph, missing_modules = _us_runtime_import_graph(tool) - # 74 = main's 70 plus spm_independence_role.py, spm_role_source.py and + # 75 = main's 70 plus spm_independence_role.py, spm_role_source.py and # spm_composition.py, reached because the pool's engine-input # projection names the SPM role as a required source input (#893), # plus asec_sources.py, whose default ASEC pool the education sidecar - # loader now defaults to. All four are classified in + # loader now defaults to, plus esi_premiums.py, whose output family + # operator_boundary registers so a raw source frame that already + # carries it is refused (#454). All five are classified in # _OTHER_US_RUNTIME_MODULES and scanned below like every other # reached module. - assert len(runtime_graph) == 74, ( - f"{tool.name} must reach the pinned 74-module runtime graph; " + assert len(runtime_graph) == 75, ( + f"{tool.name} must reach the pinned 75-module runtime graph; " f"reached {len(runtime_graph)}" ) assert not missing_modules, ( diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 5b1861afdd..c1315dedf5 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "fbf25dbae68242198972f5bef4f1fc20f65ee0769615b8d94639e6a80121f31f", + "spec_sha256": "41d89d1b74bc3aec02b8dd9c3b3004928bfb0eface7bb670827289e75ab31ab8", }, } diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py index 45f7c2c958..08c7ab2714 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py @@ -277,7 +277,7 @@ def test_shipped_baseline_plans_match_whole_file_policyengine_us( path = _write_engine_h5(builder, tmp_path, _GUARD_HOUSEHOLDS) plans, _, counts = sweep._record_requests(builder) assert counts["baseline_keys"] == { - "reform_coverage_smoke": 18, + "reform_coverage_smoke": 19, "reform_validation": 75, "demographics": 1, } diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index ef4bc8868b..0f3cd267be 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 464a658fd9..eaca029f7a 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -80,6 +80,7 @@ from microcosm.build.us_runtime import target_loss_weights as loss_weighting from microcosm.build.us_runtime.acs_local_hours import acs_local_hours_signal_gate +from microcosm.build.us_runtime.esi_premiums import refuse_unassigned_us_esi_premiums _TOOLS_DIR = Path(__file__).resolve().parent _REPO_ROOT = _TOOLS_DIR.parent @@ -502,9 +503,17 @@ def fill_reviewed_nulls( Every registered (entity, column) has its NaN filled with the pe-us variable's own default; NaN in any engine-input column NOT in the register is a hard error with a per-spine diagnostic — an artifact - defect, surfaced not filled. + defect, surfaced not filled. The ESI employer premium input is never + filled, registered or not: a null there refuses the build. """ + # microcosm#454: the ESI employer premium exists only where the + # meps_esi_premiums stage ran (the ASEC donor rows). Refuse before any + # fill: a registered null here would become a silent zero on the ACS spine. + refuse_unassigned_us_esi_premiums( + frame, consumer="ACS local release (fill_reviewed_nulls)" + ) + from policyengine_us import CountryTaxBenefitSystem from microcosm.build.us_runtime.base_pool import spine_column diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 6b64ce7059..cd6dc4063d 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -136,6 +136,9 @@ us_educator_expense_signal_gate, us_eligibility_inputs_signal_gate, us_energy_subsidy_signal_gate, + us_esi_premiums_anchor_feed_mismatches, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, us_farm_business_income_signal_gate, us_fiscal_target_exclusion_receipt, us_form_4952_election_signal_gate, @@ -1168,8 +1171,11 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: help=( "Build from a Chronicle feed whose facts (and, for an artifact " "pin, manifest) digest differs from the committed US pin in " - "us/chronicle_feed.json. Only for an explicitly reviewed " - "diagnostic run; a release build must use the pinned feed." + "us/chronicle_feed.json. Also lets through a feed whose BEA " + "B4923C facts differ from the ESI premium anchor (microcosm#454); " + "the difference is recorded in esi_premiums.json, or in the " + "dry-run report. Only for an explicitly reviewed diagnostic run; " + "a release build must use the pinned feed." ), ) parser.add_argument( @@ -1312,6 +1318,20 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "certification blocker." ), ) + parser.add_argument( + "--allow-esi-premium-gaps", + action="store_true", + help=( + "Diagnostic escape hatch (microcosm#454): record the ESI premium " + "gates — employer_sponsored_insurance_premiums absent, zero-mass, " + "missing its raw ASEC coverage columns, or off the BEA NIPA 7.8 " + "line 17 (B4923C) anchor, or a feed whose B4923C facts differ " + "from that anchor — without failing the build (for example on a " + "base built before the meps_esi_premiums stage). The failures are " + "still written to esi_premiums.json. Release builds must leave " + "this unset." + ), + ) parser.add_argument( "--allow-qrf-tail-concentration", action="store_true", @@ -7461,6 +7481,39 @@ def _engine_input_variables() -> tuple[str, ...]: return tuple(PolicyEngineUSEngine().variables()) +def _esi_premiums_signal_evidence(gate: GateResult | None) -> dict[str, object]: + """One ESI premium signal verdict for the release evidence file. + + Failures are kept verbatim so a build that waived them + (``--allow-esi-premium-gaps``) still records what was red. + """ + + if gate is None: + return {"evaluated": False} + return { + "evaluated": True, + "passed": gate.passed, + "failures": list(gate.failures), + } + + +def _esi_anchor_feed_evidence( + mismatches: Sequence[str], *, enforced: bool +) -> dict[str, object]: + """The anchor-versus-feed verdict for the release evidence file. + + ``mismatches`` is non-empty only when a diagnostic flag waived a feed + whose B4923C facts differ from the stage's anchor; they are kept verbatim + so the waived difference stays on the record. + """ + + return { + "enforced": enforced, + "matches": not mismatches, + "mismatches": list(mismatches), + } + + def _input_mass_reference_gate( base_frame: Frame, *, @@ -9976,6 +10029,7 @@ def _artifact_entry(path: str, sha: str, *, kind: str, revision: str) -> dict[st #: before the terminal gates run, so any present at manifest time are this #: run's own. US_RELEASE_GATE_EVIDENCE_FILES: dict[str, str] = { + "esi_premiums": "esi_premiums.json", "input_coverage": "input_coverage.json", "input_mass_parity": "input_mass_parity.json", "qrf_tail_concentration": "qrf_tail_concentration.json", @@ -11584,6 +11638,10 @@ def __init__( # run's own crash, never a refusal the release would share. self.stopped = False self.seconds_to_stop_point: float | None = None + # Set by _main after the feed checks; a waived B4923C difference + # (microcosm#454) is reported here, since a dry run stops before + # esi_premiums.json is written. + self.esi_anchor_feed_mismatches: list[str] = [] @classmethod def start( @@ -11616,7 +11674,13 @@ def inputs(self) -> dict[str, object]: "allow_input_mass_drift": bool(args.allow_input_mass_drift), "allow_ecps_parity_gaps": bool(args.allow_ecps_parity_gaps), "allow_input_coverage_gaps": bool(args.allow_input_coverage_gaps), + "allow_esi_premium_gaps": bool(args.allow_esi_premium_gaps), + "allow_unpinned_feed": bool(args.allow_unpinned_feed), }, + "esi_anchor_feed": _esi_anchor_feed_evidence( + self.esi_anchor_feed_mismatches, + enforced=not (args.allow_unpinned_feed or args.allow_esi_premium_gaps), + ), "margins": self.margins.to_dict(), # Taken when _main reached the stop point, before any grading; # None when the release refused earlier. @@ -12380,6 +12444,35 @@ def _check_committed_us_ledger_feed_pin( ) +def _check_us_esi_anchor_against_feed( + facts: Sequence[Mapping[str, Any]], + *, + allow_unpinned_feed: bool, + allow_esi_premium_gaps: bool, +) -> list[str]: + """Hold the ESI stage's pinned anchor to the feed's banked B4923C facts. + + microcosm#454 / d1317: the ``meps_esi_premiums`` stage scales to BEA + B4923C values it pins itself, and the pinned feed banks the same series. + On the committed pin a difference means the stage drifted from the feed, + so the build stops. A reviewed diagnostic run on another feed + (``--allow-unpinned-feed``) or with the ESI gates waived + (``--allow-esi-premium-gaps``) continues; the differences are returned + either way and written to ``esi_premiums.json``. + """ + + mismatches = us_esi_premiums_anchor_feed_mismatches(facts) + if mismatches and not (allow_unpinned_feed or allow_esi_premium_gaps): + raise SystemExit( + "error: the meps_esi_premiums anchor differs from the Chronicle " + "feed: " + + "; ".join(mismatches) + + "; pass --allow-unpinned-feed or --allow-esi-premium-gaps only " + "for an explicitly reviewed diagnostic run" + ) + return mismatches + + def _main(argv: Sequence[str] | None = None) -> int | None: global _ACTIVE_EMITTER, _ACTIVE_PROGRESS _ACTIVE_EMITTER = None @@ -12601,6 +12694,13 @@ def _main(argv: Sequence[str] | None = None) -> int | None: manifest_sha256=ledger_artifact.manifest_sha256, allow_unpinned_feed=args.allow_unpinned_feed, ) + esi_anchor_feed_mismatches = _check_us_esi_anchor_against_feed( + ledger_artifact.facts, + allow_unpinned_feed=args.allow_unpinned_feed, + allow_esi_premium_gaps=args.allow_esi_premium_gaps, + ) + if dry_run is not None: + dry_run.esi_anchor_feed_mismatches = list(esi_anchor_feed_mismatches) target_registry = compile_us_fiscal_target_registry( ledger_artifact.facts, target_period=PERIOD, @@ -14234,6 +14334,25 @@ def _main(argv: Sequence[str] | None = None) -> int | None: for failure in other_health_insurance_gate.failures ) ) + # microcosm#454: the meps_esi_premiums base stage owns the ESI employer + # premium input; refuse a base that lacks or flattens it before the solve. + esi_premiums_base_gate = us_esi_premiums_signal_gate(base_frame) + if not esi_premiums_base_gate.passed and not args.allow_esi_premium_gaps: + if telemetry is not None: + telemetry.stage( + "esi_premiums_input_gate", + status="failed", + message="ESI premium signal gate failed on the base.", + failures=list(esi_premiums_base_gate.failures), + force_upload=True, + ) + raise RuntimeError( + "Release gates failed: " + + "; ".join( + "ESI premium signal failed: " + failure + for failure in esi_premiums_base_gate.failures + ) + ) if telemetry is not None and args.input_mass_reference_h5 is not None: telemetry.stage( "input_mass_reference_gate", @@ -14996,6 +15115,25 @@ def _main(argv: Sequence[str] | None = None) -> int | None: f"Other health insurance signal failed on the export frame: {failure}" for failure in other_health_insurance_gate.failures ) + try: + esi_premiums_export_gate = us_esi_premiums_signal_gate(export_frame) + except Exception as error: + if not early_terminal_gate_failures: + raise + esi_premiums_export_gate = None + early_terminal_gate_failures.append( + "ESI premium signal evaluation crashed in degraded mode; recorded " + f"instead of masking earlier failures: {error}" + ) + if ( + esi_premiums_export_gate is not None + and not esi_premiums_export_gate.passed + and not args.allow_esi_premium_gaps + ): + early_terminal_gate_failures.extend( + f"ESI premium signal failed on the export frame: {failure}" + for failure in esi_premiums_export_gate.failures + ) if congressional_district_vintage_crosswalk_metadata is not None: compilation = { **compilation, @@ -15413,6 +15551,74 @@ def _main(argv: Sequence[str] | None = None) -> int | None: failures=list(input_coverage_gate.failures), force_upload=True, ) + # microcosm#454: the calibrated ESI premium total over the anchor's + # universe (every policyholder) against BEA NIPA 7.8 line 17 within tolerance, + # red when the column is absent or zero-mass or the raw ASEC coverage + # columns are gone. Runs on the calibrated export for BOTH the dense and + # sparse default paths; the verdict ships as the esi_premiums gate + # evidence, with the base and export signal verdicts beside it so a waived + # assignment failure stays on the record. + try: + esi_premiums_anchor_gate = us_esi_premiums_anchor_gate( + export_frame, time_period=PERIOD + ) + except Exception as exc: + if not terminal_gate_failures: + raise + terminal_gate_failures.append( + "ESI premiums failed: evaluation error under earlier gate " + f"failures: {type(exc).__name__}: {exc}" + ) + esi_premiums_anchor_gate = None + if esi_premiums_anchor_gate is not None: + esi_premiums_failed = ( + not esi_premiums_anchor_gate.passed and not args.allow_esi_premium_gaps + ) + if esi_premiums_failed: + terminal_gate_failures.extend( + f"ESI premiums failed: {failure}" + for failure in esi_premiums_anchor_gate.failures + ) + esi_premiums_path = release_dir / "esi_premiums.json" + esi_premiums_path.write_text( + json.dumps( + { + "schema_version": 3, + "enforced": not args.allow_esi_premium_gaps, + "esi_premiums": { + "passed": esi_premiums_anchor_gate.passed, + "failures": list(esi_premiums_anchor_gate.failures), + "details": dict(esi_premiums_anchor_gate.details), + }, + "anchor_feed": _esi_anchor_feed_evidence( + esi_anchor_feed_mismatches, + enforced=not ( + args.allow_unpinned_feed or args.allow_esi_premium_gaps + ), + ), + "signal": { + "base_frame": _esi_premiums_signal_evidence( + esi_premiums_base_gate + ), + "export_frame": _esi_premiums_signal_evidence( + esi_premiums_export_gate + ), + }, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + terminal_batch_telemetry.attach_artifact("esi_premiums", esi_premiums_path) + if esi_premiums_failed: + terminal_batch_telemetry.stage( + "export_dataset", + status="failed", + message="ESI premium anchor gate failed.", + failures=list(esi_premiums_anchor_gate.failures), + force_upload=True, + ) # #327: the export gate compares the calibrated export against a reference. # By default that reference is the raw pre-calibration base — but for a dense # parent built from a raw pooled-ASEC base, calibration correctly scales diff --git a/tools/build_us_meps_ic_esi_cells.py b/tools/build_us_meps_ic_esi_cells.py new file mode 100644 index 0000000000..137ce487c8 --- /dev/null +++ b/tools/build_us_meps_ic_esi_cells.py @@ -0,0 +1,606 @@ +"""Regenerate the packaged MEPS-IC employer-premium cell table (microcosm#454). + +The ``meps_esi_premiums`` source stage assigns each ESI policyholder the +MEPS-IC employer share of the premium for their coverage tier, employer size +and state (or census division for State and local government employers). +Those cells are published only as PDF tables, so this tool pins +the official AHRQ PDFs by URL, byte length and SHA-256, extracts the cells the +stage reads with ``pdftotext -layout`` (poppler), and writes them, with their +provenance, to +``packages/microcosm-build/src/microcosm/build/us_runtime/data/ +meps_ic_esi_premium_cells.json``. + +Every value is copied as published. Suppressed cells (``--``) stay ``null``: +the fallback for them is stage logic, declared in the stage manifest, not +data. The premium tables' United States rows are checked against the AHRQ +national figures they must reproduce, and a parse that does not find exactly +one row per state, division or firm-size label refuses. + +Two national blocks ride along: the share of enrollees whose coverage +required no employee contribution (Tables II.C/D/E.4.a), which turns the +average employee contribution into the contribution of those who pay one; and +the private-sector enrollment rows (Tables II.B.1, II.B.2, II.B.2.b and +II.C/D/E.4) behind the stage's active-employee cross-check. + +Run (downloads the three PDFs, about 25 MB, into ``--pdf-dir``):: + + uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic + +``--check`` regenerates in memory and fails if any published value, title or +pin in the committed file differs. It ignores the ``pdftotext`` version string +and the ``text_line`` positions, which depend on the installed poppler. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +OUTPUT_PATH = ( + REPOSITORY_ROOT + / "packages" + / "microcosm-build" + / "src" + / "microcosm" + / "build" + / "us_runtime" + / "data" + / "meps_ic_esi_premium_cells.json" +) +_MEPS_ROOT = "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/" + + +@dataclass(frozen=True) +class PinnedPdf: + key: str + url: str + size_bytes: int + sha256: str + survey_year: int + description: str + + @property + def filename(self) -> str: + return self.url.rsplit("/", 1)[-1] + + +PDFS: tuple[PinnedPdf, ...] = ( + PinnedPdf( + key="private_state_2025", + url=_MEPS_ROOT + "state/series_2/2025/ic25_iia_f.pdf", + size_bytes=12_530_005, + sha256="ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", + survey_year=2025, + description=( + "MEPS-IC 2025 Series II (private sector, by firm size and State): " + "Tables II.C/D/E.1 (average total premium per enrolled employee), " + "II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent " + "of enrollees whose coverage required no employee contribution), " + "and the enrollment rows II.B.1, II.B.2, II.B.2.b and II.C/D/E.4" + ), + ), + PinnedPdf( + key="private_state_2024", + url=_MEPS_ROOT + "state/series_2/2024/ic24_iia_f.pdf", + size_bytes=10_687_163, + sha256="4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", + survey_year=2024, + description=( + "MEPS-IC 2024 Series II (private sector, by firm size and State); " + "the United States rows age the 2024 State and local government " + "cells to 2025 and carry the 2024 private-sector enrollment" + ), + ), + PinnedPdf( + key="public_division_2024", + url=_MEPS_ROOT + "national/series_3/2024/ic24_iiia_g.pdf", + size_bytes=1_765_717, + sha256="462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79", + survey_year=2024, + description=( + "MEPS-IC 2024 Series III (State and local government, by " + "government type and size and census division): Tables " + "III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year " + "for this series (the 2025 series is not yet released)." + ), + ), +) + +TIERS = ("single", "employee_plus_one", "family") +_TIER_LETTER = {"single": "C", "employee_plus_one": "E", "family": "D"} +_SERIES_II_COLUMNS = ( + "total", + "lt10", + "10_24", + "25_99", + "100_999", + "1000plus", + "lt50", + "50plus", +) +#: The columns the stage reads. The finer firm-size columns cannot be used: +#: CPS ASEC NOEMP bands (under 10, 10-49, 50-99, 100-499, 500-999, 1000+) do +#: not nest inside MEPS-IC's 10-24 / 25-99 / 100-999 bands. +_KEPT_SERIES_II_COLUMNS = ("total", "lt50", "50plus") +_SERIES_III_COLUMNS = ( + "all_state_and_local", + "state", + "local_lt250", + "local_250_999", + "local_1000_4999", + "local_5000_9999", + "local_10000plus", +) +_KEPT_SERIES_III_COLUMNS = ("all_state_and_local", "state") +DIVISIONS = ( + "New England", + "Middle Atlantic", + "East North Central", + "West North Central", + "South Atlantic", + "East South Central", + "West South Central", + "Mountain", + "Pacific", +) +STATE_FIPS = { + "Alabama": 1, + "Alaska": 2, + "Arizona": 4, + "Arkansas": 5, + "California": 6, + "Colorado": 8, + "Connecticut": 9, + "Delaware": 10, + "District of Columbia": 11, + "Florida": 12, + "Georgia": 13, + "Hawaii": 15, + "Idaho": 16, + "Illinois": 17, + "Indiana": 18, + "Iowa": 19, + "Kansas": 20, + "Kentucky": 21, + "Louisiana": 22, + "Maine": 23, + "Maryland": 24, + "Massachusetts": 25, + "Michigan": 26, + "Minnesota": 27, + "Mississippi": 28, + "Missouri": 29, + "Montana": 30, + "Nebraska": 31, + "Nevada": 32, + "New Hampshire": 33, + "New Jersey": 34, + "New Mexico": 35, + "New York": 36, + "North Carolina": 37, + "North Dakota": 38, + "Ohio": 39, + "Oklahoma": 40, + "Oregon": 41, + "Pennsylvania": 42, + "Rhode Island": 44, + "South Carolina": 45, + "South Dakota": 46, + "Tennessee": 47, + "Texas": 48, + "Utah": 49, + "Vermont": 50, + "Virginia": 51, + "Washington": 53, + "West Virginia": 54, + "Wisconsin": 55, + "Wyoming": 56, +} + +#: United States rows each extracted table must reproduce: the AHRQ national +#: figures (MEPS-IC Research Findings and the national dashboard). A parse +#: that drifts from them refuses. +EXPECTED_NATIONAL = { + ("private_state_2025", "II.C.1"): 9_025.0, + ("private_state_2025", "II.D.1"): 26_281.0, + ("private_state_2024", "II.C.1"): 8_486.0, + ("private_state_2024", "II.C.2"): 1_789.0, + ("private_state_2024", "II.E.1"): 16_931.0, + ("private_state_2024", "II.E.2"): 4_707.0, + ("private_state_2024", "II.D.1"): 24_540.0, + ("private_state_2024", "II.D.2"): 7_216.0, +} + +_NUMBER = re.compile(r"(--|\d[\d,]*(?:\.\d+)?%?)(\s*\*)?") + + +class ExtractionError(ValueError): + """The PDF text did not parse into the reviewed table shape.""" + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def fetch(pin: PinnedPdf, pdf_dir: Path) -> Path: + """Return the verified local copy of one pinned PDF, downloading it once.""" + + pdf_dir.mkdir(parents=True, exist_ok=True) + path = pdf_dir / pin.filename + if not path.exists(): + request = urllib.request.Request( + pin.url, headers={"User-Agent": "microcosm-build/meps-ic-esi-cells"} + ) + temporary = path.with_suffix(".pdf.part") + with urllib.request.urlopen(request, timeout=300) as response: # noqa: S310 + temporary.write_bytes(response.read()) + temporary.rename(path) + size = path.stat().st_size + digest = _sha256(path) + if size != pin.size_bytes or digest != pin.sha256: + raise ExtractionError( + f"{pin.filename}: expected {pin.size_bytes} bytes / {pin.sha256}, " + f"found {size} bytes / {digest}. AHRQ reissued the file; review " + "the new tables before re-pinning." + ) + return path + + +def pdf_text(path: Path) -> list[str]: + completed = subprocess.run( + ["pdftotext", "-layout", str(path), "-"], + check=True, + capture_output=True, + text=True, + ) + return [line.lstrip("\f") for line in completed.stdout.splitlines()] + + +def pdftotext_version() -> str: + completed = subprocess.run( + ["pdftotext", "-v"], check=True, capture_output=True, text=True + ) + return (completed.stderr or completed.stdout).splitlines()[0].strip() + + +def _table_block(lines: list[str], table_id: str) -> tuple[int, list[str]]: + head = re.compile(rf"^Table {re.escape(table_id)} (Average|Percent|Number)") + starts = [ + index + for index, line in enumerate(lines) + if head.match(line) and "Standard errors" not in line + ] + if len(starts) != 1: + raise ExtractionError(f"Table {table_id}: found {len(starts)} headers.") + start = starts[0] + end = next( + ( + index + for index in range(start + 1, len(lines)) + if lines[index].startswith("Table ") + ), + len(lines), + ) + return start, lines[start:end] + + +def _row_values(block: list[str], label: str, width: int, table_id: str): + pattern = re.compile(rf"^\s*{re.escape(label)}\s{{2,}}") + rows = [line for line in block if pattern.match(line)] + if len(rows) != 1: + raise ExtractionError(f"Table {table_id}: {label!r} matched {len(rows)} rows.") + tokens = _NUMBER.findall(rows[0].split(label, 1)[1]) + if len(tokens) != width: + raise ExtractionError( + f"Table {table_id}: {label!r} has {len(tokens)} values, not {width}: " + f"{rows[0]!r}." + ) + values: list[float | None] = [] + unreliable: list[bool] = [] + for token, star in tokens: + values.append( + None if token == "--" else float(token.replace(",", "").replace("%", "")) + ) + unreliable.append(bool(star.strip())) + return values, unreliable + + +def _table_year(block: list[str], table_id: str) -> int: + match = re.search(r"United States, (\d{4})", " ".join(block[:3])) + if match is None: + raise ExtractionError(f"Table {table_id}: no survey year in the title.") + return int(match.group(1)) + + +def _state_divisions(block: list[str]) -> dict[str, str]: + """State -> census division from the table's own division headings.""" + + division = None + membership: dict[str, str] = {} + for line in block: + stripped = line.strip() + if stripped.endswith(":") and stripped[:-1] in DIVISIONS: + division = stripped[:-1] + continue + for state in STATE_FIPS: + if re.match(rf"^{re.escape(state)}\s{{2,}}", stripped): + if division is None: + raise ExtractionError(f"{state} precedes any division heading.") + membership[state] = division + if set(membership) != set(STATE_FIPS): + raise ExtractionError( + "Division headings did not place every state: missing " + f"{sorted(set(STATE_FIPS) - set(membership))}." + ) + return membership + + +def _series_ii_table(lines, pin: PinnedPdf, table_id: str) -> dict: + start, block = _table_block(lines, table_id) + if _table_year(block, table_id) != pin.survey_year: + raise ExtractionError(f"Table {table_id} is not survey year {pin.survey_year}.") + rows = {} + for label in ("United States", *STATE_FIPS): + # Labels anchor at the line start, so "Virginia" cannot match the + # "West Virginia" row nor "Kansas" the "Arkansas" row. + values, unreliable = _row_values( + block, label, len(_SERIES_II_COLUMNS), table_id + ) + by_column = dict(zip(_SERIES_II_COLUMNS, values, strict=True)) + flags = dict(zip(_SERIES_II_COLUMNS, unreliable, strict=True)) + key = "US" if label == "United States" else f"{STATE_FIPS[label]:02d}" + rows[key] = {column: by_column[column] for column in _KEPT_SERIES_II_COLUMNS} + starred = [column for column in _KEPT_SERIES_II_COLUMNS if flags[column]] + if starred: + rows[key]["unreliable"] = starred + expected = EXPECTED_NATIONAL.get((pin.key, table_id)) + if expected is not None and rows["US"]["total"] != expected: + raise ExtractionError( + f"Table {table_id} ({pin.survey_year}) United States total is " + f"{rows['US']['total']}, not the published {expected}." + ) + return {"text_line": start + 1, "title": block[0].strip(), "rows": rows} + + +def _series_iii_table(lines, pin: PinnedPdf, table_id: str) -> dict: + start, block = _table_block(lines, table_id) + if _table_year(block, table_id) != pin.survey_year: + raise ExtractionError(f"Table {table_id} is not survey year {pin.survey_year}.") + rows = {} + for label in ("United States", *DIVISIONS): + values, _unreliable = _row_values( + block, label, len(_SERIES_III_COLUMNS), table_id + ) + by_column = dict(zip(_SERIES_III_COLUMNS, values, strict=True)) + kept = {column: by_column[column] for column in _KEPT_SERIES_III_COLUMNS} + if any(value is None for value in kept.values()): + raise ExtractionError(f"Table {table_id}: {label} has a suppressed cell.") + rows["US" if label == "United States" else label] = kept + return {"text_line": start + 1, "title": block[0].strip(), "rows": rows} + + +def _full_title(block: list[str]) -> str: + """A table title through its ``United States, `` line (it may wrap).""" + + title = "" + for line in block[:4]: + title = f"{title} {line.strip()}".strip() + if re.search(r"United States, \d{4}$", title): + return title + raise ExtractionError(f"Table title does not end in a survey year: {block[0]!r}.") + + +def _national_row(lines, pin: PinnedPdf, table_id: str) -> dict: + """The United States row of one Series II table, kept firm-size columns.""" + + start, block = _table_block(lines, table_id) + if _table_year(block, table_id) != pin.survey_year: + raise ExtractionError(f"Table {table_id} is not survey year {pin.survey_year}.") + values, unreliable = _row_values( + block, "United States", len(_SERIES_II_COLUMNS), table_id + ) + by_column = dict(zip(_SERIES_II_COLUMNS, values, strict=True)) + flags = dict(zip(_SERIES_II_COLUMNS, unreliable, strict=True)) + kept = {column: by_column[column] for column in _KEPT_SERIES_II_COLUMNS} + bad = [ + column + for column in _KEPT_SERIES_II_COLUMNS + if kept[column] is None or flags[column] + ] + if bad: + raise ExtractionError( + f"Table {table_id}: United States {bad} suppressed or flagged unreliable." + ) + return {"text_line": start + 1, "title": _full_title(block), **kept} + + +def _no_contribution_shares(lines, pin: PinnedPdf) -> dict: + """Percent of enrollees, by tier, whose coverage required no contribution. + + National by firm size only: most State cells of these tables are flagged + unreliable or suppressed. + """ + + return { + tier: _national_row(lines, pin, f"II.{_TIER_LETTER[tier]}.4.a") + for tier in TIERS + } + + +def _private_enrollment(lines, pin: PinnedPdf) -> dict: + """National private-sector enrollment rows for the active-employee check.""" + + return { + "employees": _national_row(lines, pin, "II.B.1"), + "offer_percent": _national_row(lines, pin, "II.B.2"), + "enrolled_percent": _national_row(lines, pin, "II.B.2.b"), + "tier_share_percent": { + tier: _national_row(lines, pin, f"II.{_TIER_LETTER[tier]}.4") + for tier in TIERS + }, + } + + +def build_payload(pdf_dir: Path) -> dict: + texts = {pin.key: pdf_text(fetch(pin, pdf_dir)) for pin in PDFS} + private_2025 = { + tier: { + "premium": _series_ii_table( + texts["private_state_2025"], + PDFS[0], + f"II.{_TIER_LETTER[tier]}.1", + ), + "employee_contribution": _series_ii_table( + texts["private_state_2025"], + PDFS[0], + f"II.{_TIER_LETTER[tier]}.2", + ), + } + for tier in TIERS + } + private_2024_national = {} + for tier in TIERS: + private_2024_national[tier] = {} + for measure, suffix in (("premium", "1"), ("employee_contribution", "2")): + table = _series_ii_table( + texts["private_state_2024"], + PDFS[1], + f"II.{_TIER_LETTER[tier]}.{suffix}", + ) + private_2024_national[tier][measure] = { + "text_line": table["text_line"], + "total": table["rows"]["US"]["total"], + } + public_2024 = { + tier: { + "premium": _series_iii_table( + texts["public_division_2024"], + PDFS[2], + f"III.{_TIER_LETTER[tier]}.1", + ), + "employee_contribution": _series_iii_table( + texts["public_division_2024"], + PDFS[2], + f"III.{_TIER_LETTER[tier]}.2", + ), + } + for tier in TIERS + } + _start, division_block = _table_block(texts["private_state_2025"], "II.C.1") + divisions = _state_divisions(division_block) + return { + "schema_version": 2, + "issue": "PolicyEngine/microcosm#454", + "description": ( + "MEPS-IC average total premium and average employee contribution " + "per enrolled employee, by coverage tier, for the " + "meps_esi_premiums source stage: private-sector cells by State and " + "firm size (2025), State and local government cells by census " + "division (2024, the latest published year), the 2024 private " + "national rows that age the government cells, the 2025 national " + "share of enrollees whose coverage required no employee " + "contribution, and the 2024 and 2025 national private-sector " + "enrollment rows. Values are copied as published; null marks a " + "cell AHRQ suppressed ('--'). Dollar values are annual; percent " + "values are percents; employee counts are persons." + ), + "generator": "tools/build_us_meps_ic_esi_cells.py", + "extraction": { + "method": "pdftotext -layout, one row per label, refusing any " + "label that does not match exactly one line", + "pdftotext": pdftotext_version(), + }, + "sources": { + pin.key: { + "url": pin.url, + "size_bytes": pin.size_bytes, + "sha256": pin.sha256, + "survey_year": pin.survey_year, + "publisher": ( + "Agency for Healthcare Research and Quality, Center for " + "Financing, Access and Cost Trends, Medical Expenditure " + "Panel Survey-Insurance Component" + ), + "description": pin.description, + } + for pin in PDFS + }, + "tiers": list(TIERS), + "state_census_division": { + f"{STATE_FIPS[state]:02d}": divisions[state] for state in STATE_FIPS + }, + "private_state_2025": private_2025, + "private_national_2024": private_2024_national, + "public_division_2024": public_2024, + "no_contribution_share_2025": _no_contribution_shares( + texts["private_state_2025"], PDFS[0] + ), + "private_enrollment_national": { + "2024": _private_enrollment(texts["private_state_2024"], PDFS[1]), + "2025": _private_enrollment(texts["private_state_2025"], PDFS[0]), + }, + } + + +def render(payload: dict) -> str: + return json.dumps(payload, indent=1, sort_keys=True) + "\n" + + +#: Provenance that depends on the installed poppler, not on the PDFs: the +#: ``pdftotext`` version string and the text line each table was found on. +_ENVIRONMENT_KEYS = frozenset({"pdftotext", "text_line"}) + + +def published_values(payload): + """The payload without poppler-dependent provenance, for ``--check``.""" + + if isinstance(payload, dict): + return { + key: published_values(value) + for key, value in payload.items() + if key not in _ENVIRONMENT_KEYS + } + if isinstance(payload, list): + return [published_values(item) for item in payload] + return payload + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--pdf-dir", + type=Path, + required=True, + help="Directory caching the pinned AHRQ PDFs (downloaded when absent).", + ) + parser.add_argument( + "--check", + action="store_true", + help="Fail if the committed cell table differs from a regeneration.", + ) + args = parser.parse_args(argv) + payload = build_payload(args.pdf_dir) + rendered = render(payload) + if args.check: + committed = json.loads(OUTPUT_PATH.read_text()) + if published_values(committed) != published_values(payload): + print(f"{OUTPUT_PATH} is stale; regenerate it.", file=sys.stderr) + return 1 + print(f"{OUTPUT_PATH} is current.") + return 0 + OUTPUT_PATH.write_text(rendered) + print(f"Wrote {OUTPUT_PATH}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index b5f8663ea1..1ec4cafa1b 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -105,6 +105,7 @@ us_educator_expense_signal_gate, us_eligibility_inputs_signal_gate, us_energy_subsidy_signal_gate, + us_esi_premiums_signal_gate, us_farm_business_income_signal_gate, us_form_4952_election_signal_gate, us_geography_ladder_assignment_summary, @@ -136,6 +137,7 @@ with_us_education_inputs, with_us_eligibility_inputs, with_us_energy_subsidy_input, + with_us_esi_premium_inputs, with_us_housing_inputs, with_us_immigration_inputs, with_us_medicare_take_up_input, @@ -224,6 +226,7 @@ "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", + "with_us_esi_premium_inputs", "with_us_immigration_inputs", ), ), @@ -1217,6 +1220,17 @@ def _run_all( seed=args.seed, time_period=args.target_year, ) + base = with_us_esi_premium_inputs( + base, + seed=args.seed, + time_period=args.target_year, + ) + esi_premiums_gate = us_esi_premiums_signal_gate(base) + if not esi_premiums_gate.passed: + raise SystemExit( + "ESI premium signal gate failed before support cloning:\n " + + "\n ".join(esi_premiums_gate.failures) + ) base = with_us_immigration_inputs( base, seed=args.seed, @@ -1515,6 +1529,14 @@ def _run_all( "Education-input signal gate failed:\n " + "\n ".join(education_inputs_gate.failures) ) + # The ESI premium inputs ride the support clone unchanged; re-prove clone + # agreement and the cell multiples on the cloned frame. + esi_premiums_post_clone_gate = us_esi_premiums_signal_gate(imputed) + if not esi_premiums_post_clone_gate.passed: + raise SystemExit( + "ESI premium signal gate failed after support cloning:\n " + + "\n ".join(esi_premiums_post_clone_gate.failures) + ) _observe_frame_boundary(boundary_observer, "education_inputs_post_clone", imputed) congressional_district_assignment = {"applied": False} if args.assign_congressional_districts: @@ -1725,6 +1747,16 @@ def _run_all( "failures": list(pregnancy_gate.failures), "details": dict(pregnancy_gate.details), }, + "esi_premiums_signal": { + "passed": esi_premiums_gate.passed, + "failures": list(esi_premiums_gate.failures), + "details": dict(esi_premiums_gate.details), + }, + "esi_premiums_post_clone_signal": { + "passed": esi_premiums_post_clone_gate.passed, + "failures": list(esi_premiums_post_clone_gate.failures), + "details": dict(esi_premiums_post_clone_gate.details), + }, "wic_claim_signal": { "passed": wic_claim_gate.passed, "failures": list(wic_claim_gate.failures), @@ -2328,6 +2360,15 @@ def _pre_clone_enrichment_stage( seed=args.seed, time_period=args.target_year, ) + base = with_us_esi_premium_inputs( + base, + seed=args.seed, + time_period=args.target_year, + ) + signals["esi_premiums_signal"] = _checked_gate_payload( + us_esi_premiums_signal_gate(base), + "ESI premium signal gate failed before support cloning", + ) base = with_us_immigration_inputs( base, seed=args.seed, @@ -2874,6 +2915,10 @@ def _post_qrf_frame_stage( us_education_inputs_signal_gate(frame), "Education-input signal gate failed", ) + signals["esi_premiums_post_clone_signal"] = _checked_gate_payload( + us_esi_premiums_signal_gate(frame), + "ESI premium signal gate failed after support cloning", + ) elif stage == "congressional_district_assignment": assignment: dict[str, object] = {"applied": False} if args.assign_congressional_districts: @@ -3021,6 +3066,8 @@ def _export_staged_result( "weeks_unemployed_signal", "eligibility_inputs_signal", "pregnancy_signal", + "esi_premiums_signal", + "esi_premiums_post_clone_signal", "wic_claim_signal", "educator_expense_signal", "form_4952_election_signal", diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index be6f9f56f2..b3bc625863 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -113,6 +113,20 @@ # from today's artifacts is the intended red gate (the #368 SSI-asset # pattern, extended by #462 to the capital-gain-distributions route). POST_REFERENCE_COLUMN_NOTES = { + "employer_sponsored_insurance_premiums": ( + "Employer share of the premium for employed current ESI policyholders " + "(PolicyEngine/microcosm#454), written by the meps_esi_premiums " + "base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / " + "NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person " + "files) and MEPS-IC tier x firm-size x State cells, scaled over every " + "policyholder to BEA NIPA 7.8 line 17 (B4923C). A release must persist it: CBO " + "household market income (gov.household.cbo_market_income_additions) " + "and every ESI-to-wages reform bind through it. Required with NO " + "reviewed exclusion; absent on pre-#454 bases by construction — the " + "intended red gate until a base built with the stage ships. The " + "release also holds the anchor-universe total to the anchor " + "(us_esi_premiums_anchor_gate)." + ), "schedule_d_capital_gain_distributions": ( "Schedule D line 13 route leg of the #282 capital-gain-distributions " "split (memo component of long_term_capital_gains, written by the " @@ -229,6 +243,8 @@ OTHER_HEALTH_INSURANCE_INPUTS = ("other_health_insurance_premiums",) +ESI_PREMIUM_INPUTS = ("employer_sponsored_insurance_premiums",) + PRIOR_YEAR_INCOME_INPUTS = ( "self_employment_income_last_year", "previous_year_income_available", @@ -300,6 +316,7 @@ *WIC_CLAIM_INPUTS, *EDUCATOR_EXPENSE_INPUTS, *OTHER_HEALTH_INSURANCE_INPUTS, + *ESI_PREMIUM_INPUTS, *FARM_BUSINESS_INCOME_INPUTS, *SCF_NET_WORTH_INPUTS, *SIPP_VEHICLE_INPUTS, @@ -347,6 +364,32 @@ #: a floor far below the plausible effect but far above simulation noise, so a #: structural $0 fails while a real (even conservative) score passes. REFORM_COVERAGE_PROBES = [ + { + "id": "employer_sponsored_insurance_premium_neutralization", + "name": "Employer-sponsored insurance premium neutralization", + "parameter_changes": {}, + "neutralized_variable": "employer_sponsored_insurance_premiums", + "budget_measure": "cbo_household_market_income", + "period": 2024, + "effect_direction": "baseline_minus_reform", + "expected_sign": "positive", + "binding_inputs": ["employer_sponsored_insurance_premiums"], + "min_abs_effect": 500_000_000_000.0, + "reason": ( + "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums " + "to CBO household market income through " + "gov.household.cbo_market_income_additions, its only consumer, so " + "neutralizing it lowers cbo_household_market_income by the " + "weighted employer-premium total: about $864 billion at 2024 (the " + "employed policyholders' part of BEA NIPA 7.8 line 17's $977.0 " + "billion, the meps_esi_premiums anchor; the stage measured $863.9 " + "billion over 74.0 million weighted employed policyholders on the " + "default 2023-2025 pool, before calibration). The floor sits at " + "about half of that. A structural zero means the column was " + "dropped or the stage never ran." + ), + "issue": "PolicyEngine/microcosm#454", + }, { "id": "prior_year_self_employment_neutralization", "name": "Prior-year self-employment support neutralization", diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 0b8548ebe3..6219fc10af 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -61,7 +61,7 @@ MANIFEST_PATH = US_PACKAGE_DIR / "target_parity_manifest.json" FEED_FAMILIES_PATH = US_PACKAGE_DIR / "target_parity_feed_families.json" -DEFAULT_FEED_NAME = "consumer_facts_us_51aa40f.jsonl" +DEFAULT_FEED_NAME = "consumer_facts_us_cf9ece2.jsonl" DEFAULT_FEED_PATH = ( Path.home() / "PolicyEngine" / "_buildh-runtime" / "inputs" / DEFAULT_FEED_NAME ) @@ -239,58 +239,129 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: ), ) -_ESI_ANCHOR_EXCLUSION = ( - "not_modeled", - "NHE employer-contribution ESI premium aggregate: a retired us-data " - "calibration target (loss.py 1_002.9e9, removed at 9f320d2c) whose " - "anchor v9.2 re-banks (2026-07-21) for microcosm#454's future gate. The pe-us input it would anchor " - "(employer_sponsored_insurance_premiums, the employer-paid share per " - "the variable's own documentation) has no live producer in the " - "hermetic lineage, so a target would bind against a structural zero. " - "The all-employer series is the future anchor (pe-us covers government " - "and private employees alike); the private-employer subset rides as its " - "cross-check.", - "ecps_parity_known_gaps.json entry employer_sponsored_insurance_" - "premiums (NOW_* source-unavailability evidence); microcosm#454 " - "descope comment 2026-07-21", +_ESI_ORIGIN = ( + "a REAL us-data calibration target: utils/loss.py pinned " + "employer_sponsored_insurance_premiums at 1_002.9e9 (this series, the NIPA " + "7.8 employer-contribution aggregate, before BEA's 2026-09-30 revision; " + "verified at archived commit 2e044571, loss.py line 33, with " + "etl_national_targets carrying the row), removed at 9f320d2c ('Remove " + "survey-based national calibration targets', PR #831). The input itself " + "was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* " + "policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581)." +) + +_ESI_NHE_ORIGIN = ( + "not itself a us-data calibration target: the retired pipeline calibrated " + "employer_sponsored_insurance_premiums to the BEA NIPA 7.8 series (family " + "bea_nipa.private_group_health_insurance; utils/loss.py 1_002.9e9 at " + "archived commit 2e044571, removed at 9f320d2c, PR #831). The v9.2 feed " + "banked the NHE Table 24 series for the microcosm#454 gate before " + "decision d1317 moved the stage's anchor to BEA." +) + +_ESI_BEA_ANCHOR_EXCLUSION = ( + "deferred", + "BEA NIPA Table 7.8 line 17 (B4923C), employer contributions for private " + "group health insurance: the scale anchor of the meps_esi_premiums base " + "stage (microcosm#454, decision d1317). The stage prices every current ESI " + "policyholder from MEPS-IC, scales that universe to this fact (BEA State " + "Personal Income methodology, December 2025, paragraphs 3.18-3.19: " + "contributions for active and retired employees) and writes the pe-us " + "input (employer_sponsored_insurance_premiums) for employed policyholders " + "only. So the fact is not a sum target on the column: the column carries " + "about 88% of it and the rest is priced, not exported. Validation-gated " + "instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total " + "at release weights and holds it within 5% of this fact, and the release " + "tool refuses a feed whose values differ from the stage's pinned anchor " + "(us_esi_premiums_anchor_feed_mismatches).", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M " + "weighted current policyholders; anchor-universe total $977.0B at scale " + "factor 0.971; employed column $863.9B (88.4%); other policyholders " + "$113.2B; private-sector part of the column $665.6B against MEPS-IC's own " + "private active-employee total of $668.6B (2024)", _fence( - origin=( - "a REAL us-data calibration target: utils/loss.py pinned " - "employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA " - "7.8 employer-contribution aggregate; verified at archived " - "commit 2e044571, loss.py line 33, with etl_national_targets " - "carrying the row), removed at 9f320d2c ('Remove survey-based " - "national calibration targets', PR #831). The input itself was " - "imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* " - "policyholder fields, archived 42ed5d45 cps.py " - "L197-271/L1575-1581). The v9.2 feed facts re-bank that same " - "anchor for the future gate, entered deliberately ahead of the " - "producer." + origin=_ESI_ORIGIN, + purpose=( + "anchor the CBO market-income ESI component once the column has a " + "live producer. The BEA annual update of 2026-09-30 revised CY2024 " + "from the $1,002.9B the retired target used to $977.034B." + ), + verdict_basis=( + "deferred, not dropped: the producer is live and scaled to this " + "fact, but the column is the employed policyholders' part of it, " + "so the fact cannot be compiled as a sum target on the column. " + "Release validation (us_esi_premiums_anchor_gate, both dense and " + "sparse exports) holds the anchor-universe total to it. Whether " + "and how to calibrate the column is decision d1264." ), + ), +) + +_ESI_NHE_EXCLUSION = ( + "deferred", + "NHE Table 24 employer contribution to ESI premiums, a second estimate of " + "the concept the meps_esi_premiums stage scales to (microcosm#454): " + "employer contributions for active employees, COBRA enrollees and " + "retirees (NHEA methodology paper 2024, p. 29). Decision d1317 chose BEA " + "NIPA 7.8 line 17 as the stage's anchor instead; this fact is 7.2% above " + "it for CY2024. It is neither compiled nor gated: " + "us_esi_premiums_anchor_gate records the anchor-universe total against it " + "beside the verdict (cross_check_ratio).", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: " + "anchor-universe total $977.0B, 0.933 of NHE's $1,047.0B; scaled to NHE " + "instead, the employed column would be $925.7B", + _fence( + origin=_ESI_NHE_ORIGIN, + purpose=( + "a cross-check on the ESI anchor's level: an independent estimate " + "of the same active-plus-retired concept from the national health " + "accounts." + ), + verdict_basis=( + "deferred, not dropped: the stage's anchor is BEA (d1317), and " + "this fact, like it, includes retirees, so it is not a sum target " + "on the employed column. It is recorded beside the anchor " + "verdict. Whether and how to calibrate the column is decision " + "d1264." + ), + ), +) + +_ESI_PRIVATE_EMPLOYER_EXCLUSION = ( + "deferred", + "NHE Table 24 private-employer contribution to ESI premiums, the private " + "part of the NHE all-employer fact (microcosm#454). It is neither " + "compiled nor gated: the stage scales one factor to BEA NIPA 7.8 line 17 " + "over every current ESI policyholder, and nothing holds the private-sector " + "part of the column to this fact, which also counts private retirees and " + "COBRA enrollees. The private part is instead recorded beside the anchor " + "verdict against MEPS-IC's own active-employee total " + "(us_esi_premiums_anchor_gate, private_active_cross_check).", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: " + "private-sector part of the column $665.6B; MEPS-IC private " + "active-employee total $668.6B (2024)", + _fence( + origin=_ESI_NHE_ORIGIN, purpose=( - "anchor a validation gate for the CBO market-income ESI " - "component once the column has a live producer: NIPA 7.8 " - "employer contributions for group health insurance and the " - "NHEA sponsor-of-funds employer share are same-concept " - "estimates (~4 percent apart, CY2024)." + "a sector cross-check that private employers' share of the ESI " + "total is in range." ), verdict_basis=( - "us-data-targeted, so under the absolute rule this compiles " - "UNLESS source-absent — and the producer source is absent: the " - "declared meps_esi_premiums stage cannot execute because all " - "three required NOW_* fields are missing from the sha-pinned " - "2022-2024 ASEC h5 inputs (recorded parity known-gap), so a " - "target would bind against a structural zero. Compiles when " - "microcosm#454 restores the source fields or lands a validated " - "PHIP_VAL+MEPS re-derivation; the anchor value is already " - "banked in the feed." + "deferred, not dropped: no gate reads this fact. The private part " + "of the column is compared with MEPS-IC's active-employee total " + "instead, because this fact includes private retirees and COBRA " + "enrollees. Whether and how to calibrate the column is decision " + "d1264." ), ), ) _FAMILY_EXCLUSIONS: dict[str, tuple[str, str, str, dict[str, str]]] = { - "cms_nhe.esi_employer_contribution_premiums": _ESI_ANCHOR_EXCLUSION, - "cms_nhe.esi_private_employer_contribution_premiums": _ESI_ANCHOR_EXCLUSION, + "bea_nipa.private_group_health_insurance": _ESI_BEA_ANCHOR_EXCLUSION, + "cms_nhe.esi_employer_contribution_premiums": _ESI_NHE_EXCLUSION, + "cms_nhe.esi_private_employer_contribution_premiums": ( + _ESI_PRIVATE_EMPLOYER_EXCLUSION + ), "jct.obbba_title_vii": _JCT_OBBBA_NO_TAX_ANCHOR_EXCLUSION, "bea_nipa.personal_interest_income": ( "macro_control_total", diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 5dc45835db..1eda413959 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index c066d573ce..16a8e339db 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7" + "c1e2a118a0d53db0bb628b14357a2c86db2956332f5b359c9092764f9c451a95" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1]