From 9fb23e30d02527886245a5aeaa8c5594082c8474 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 12:04:20 -0400 Subject: [PATCH 1/8] Vendor the MEPS-IC employer-premium cells and their extractor (#454) AHRQ publishes the MEPS-IC State tables only as PDF. The tool pins four PDFs by URL, byte length and SHA-256, extracts the cells the meps_esi_premiums stage reads with pdftotext, and writes them with their provenance to us_runtime/data/meps_ic_esi_premium_cells.json: - 2025 Series II, private sector by State and firm size: average total premium and employee contribution, single / employee-plus-one / family; - 2024 Series III, State and local government by census division (2025 is not published), plus the 2024 private national rows that age it; - 2025 Series I, pretax-contribution and health-insurance offer rates by firm size. Values are copied as published; a suppressed cell stays null. The tool refuses a reissued PDF, a label that matches no row or two, a table for another survey year, and a United States row that differs from the AHRQ national figure. --check fails if the committed file is stale. Co-Authored-By: Claude Opus 5.5 --- .../data/MEPS_IC_ESI_PREMIUM_CELLS.md | 41 + .../data/meps_ic_esi_premium_cells.json | 2101 +++++++++++++++++ tools/build_us_meps_ic_esi_cells.py | 548 +++++ 3 files changed, 2690 insertions(+) create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json create mode 100644 tools/build_us_meps_ic_esi_cells.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md new file mode 100644 index 0000000000..14d5a545a4 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md @@ -0,0 +1,41 @@ +# MEPS-IC employer-premium cells + +`meps_ic_esi_premium_cells.json` holds the MEPS-IC averages the +`meps_esi_premiums` stage assigns to employed ESI policyholders +(`microcosm.build.us_runtime.esi_premiums`, +[PolicyEngine/microcosm#454](https://github.com/PolicyEngine/microcosm/issues/454); +method in `docs/us-esi-employer-premiums.md`). + +AHRQ publishes the State tables only as PDF. Each value is copied as +published; `null` marks a cell AHRQ suppressed (`--`). The fallback for a +suppressed cell is stage logic, not data. + +| Key | Source | Contents | +|---|---|---| +| `private_state_2025` | MEPS-IC 2025 Series II, Tables II.C.1/2, II.D.1/2, II.E.1/2 | Average total premium and average employee contribution per enrolled employee at private-sector establishments, single / family / employee-plus-one, by State, for all firms, under 50 and 50 or more employees | +| `private_national_2024` | MEPS-IC 2024 Series II, United States rows | The same six national averages a year earlier; they age the government cells | +| `public_division_2024` | MEPS-IC 2024 Series III, Tables III.C.1/2, III.D.1/2, III.E.1/2 | The same averages for State and local government jobs, by census division, for all governments and State governments. 2024 is the latest year AHRQ has published for this series | +| `pretax_contribution_2025` | MEPS-IC 2025 Series I, Tables I.A.2 and I.A.2.j | Percent of private-sector establishments that offer health insurance, and that offer pretax employee contributions to it, by firm size | +| `state_census_division` | the division headings of Table II.C.1 | State FIPS to census division | + +`sources` pins each PDF by URL, byte length and SHA-256. + +## How it is built + +``` +uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic +``` + +The tool downloads the four pinned PDFs (about 34 MB) into `--pdf-dir`, +refuses any whose length or digest differs from its pin, extracts text with +`pdftotext -layout` (poppler) and reads one row per State, division or +firm-size label. It refuses a label that matches no line or more than one, a +row with the wrong number of values, a table whose title names another survey +year, and a United States row that differs from the AHRQ national figure it +must reproduce (for example single $9,025 and family $26,281 in 2025; the +2024 rows of MEPS-IC Research Findings #54). `--check` regenerates in memory +and fails if the committed file differs. + +The stage pins this file's own SHA-256 (`esi_premiums._CELLS_SHA256`) and +refuses to load any other bytes, so a regeneration must be re-pinned there +and in `us/source_stages.json`. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json b/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json new file mode 100644 index 0000000000..6e409ee3c2 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json @@ -0,0 +1,2101 @@ +{ + "description": "MEPS-IC average total premium and average employee contribution per enrolled employee, by coverage tier, for the meps_esi_premiums source stage: private-sector cells by State and firm size (2025), State and local government cells by census division (2024, the latest published year), the 2024 private national rows that age the government cells, and the 2025 firm-size pretax-contribution offer rates. Values are copied as published; null marks a cell AHRQ suppressed ('--'). Dollar values are annual; percent values are percents.", + "extraction": { + "method": "pdftotext -layout, one row per label, refusing any label that does not match exactly one line", + "pdftotext": "pdftotext version 26.09.0" + }, + "generator": "tools/build_us_meps_ic_esi_cells.py", + "issue": "PolicyEngine/microcosm#454", + "pretax_contribution_2025": { + "offer_title": "Table I.A.2 Percent of private-sector establishments that offer health insurance by firm size and selected characteristics: United States,", + "rows": { + "50plus": { + "health_insurance_offer_percent": 94.8, + "pretax_contribution_offer_percent": 87.8 + }, + "lt50": { + "health_insurance_offer_percent": 32.1, + "pretax_contribution_offer_percent": 15.9 + }, + "total": { + "health_insurance_offer_percent": 49.2, + "pretax_contribution_offer_percent": 35.5 + } + }, + "text_line": 2376, + "title": "Table I.A.2.j Percent of private-sector establishments that offer tax-advantaged benefits for contributions to health insurance, Flexible" + }, + "private_national_2024": { + "employee_plus_one": { + "employee_contribution": { + "text_line": 16845, + "total": 4707.0 + }, + "premium": { + "text_line": 16347, + "total": 16931.0 + } + }, + "family": { + "employee_contribution": { + "text_line": 13493, + "total": 7216.0 + }, + "premium": { + "text_line": 12495, + "total": 24540.0 + } + }, + "single": { + "employee_contribution": { + "text_line": 9645, + "total": 1789.0 + }, + "premium": { + "text_line": 8647, + "total": 8486.0 + } + } + }, + "private_state_2025": { + "employee_plus_one": { + "employee_contribution": { + "rows": { + "01": { + "50plus": 3878.0, + "lt50": 8912.0, + "total": 4545.0 + }, + "02": { + "50plus": 4583.0, + "lt50": null, + "total": 4584.0 + }, + "04": { + "50plus": 4529.0, + "lt50": 3563.0, + "total": 4366.0 + }, + "05": { + "50plus": 5194.0, + "lt50": 3252.0, + "total": 4874.0 + }, + "06": { + "50plus": 4999.0, + "lt50": 6153.0, + "total": 5159.0 + }, + "08": { + "50plus": 4346.0, + "lt50": 6320.0, + "total": 4543.0, + "unreliable": [ + "lt50" + ] + }, + "09": { + "50plus": 3695.0, + "lt50": 5396.0, + "total": 3857.0 + }, + "10": { + "50plus": 5279.0, + "lt50": 3689.0, + "total": 5084.0, + "unreliable": [ + "lt50" + ] + }, + "11": { + "50plus": 5224.0, + "lt50": 3685.0, + "total": 5001.0 + }, + "12": { + "50plus": 4751.0, + "lt50": 4395.0, + "total": 4714.0 + }, + "13": { + "50plus": 4769.0, + "lt50": 5660.0, + "total": 4853.0, + "unreliable": [ + "lt50" + ] + }, + "15": { + "50plus": 4173.0, + "lt50": 2472.0, + "total": 3806.0, + "unreliable": [ + "lt50" + ] + }, + "16": { + "50plus": 4820.0, + "lt50": 6590.0, + "total": 5078.0 + }, + "17": { + "50plus": 5238.0, + "lt50": 3694.0, + "total": 5016.0 + }, + "18": { + "50plus": 4226.0, + "lt50": 6270.0, + "total": 4424.0 + }, + "19": { + "50plus": 4538.0, + "lt50": 3575.0, + "total": 4408.0 + }, + "20": { + "50plus": 4954.0, + "lt50": 4195.0, + "total": 4862.0 + }, + "21": { + "50plus": 3503.0, + "lt50": null, + "total": 3676.0 + }, + "22": { + "50plus": 4682.0, + "lt50": 7519.0, + "total": 5033.0 + }, + "23": { + "50plus": 4353.0, + "lt50": 8221.0, + "total": 4829.0, + "unreliable": [ + "lt50" + ] + }, + "24": { + "50plus": 4821.0, + "lt50": 4989.0, + "total": 4840.0 + }, + "25": { + "50plus": 4785.0, + "lt50": 4715.0, + "total": 4778.0 + }, + "26": { + "50plus": 4232.0, + "lt50": 9482.0, + "total": 5251.0, + "unreliable": [ + "lt50" + ] + }, + "27": { + "50plus": 5760.0, + "lt50": 8314.0, + "total": 6043.0 + }, + "28": { + "50plus": 4317.0, + "lt50": 3552.0, + "total": 4217.0 + }, + "29": { + "50plus": 4795.0, + "lt50": 6982.0, + "total": 4993.0 + }, + "30": { + "50plus": 3495.0, + "lt50": 4138.0, + "total": 3674.0, + "unreliable": [ + "lt50" + ] + }, + "31": { + "50plus": 4593.0, + "lt50": 4432.0, + "total": 4577.0 + }, + "32": { + "50plus": 4324.0, + "lt50": 6257.0, + "total": 4498.0 + }, + "33": { + "50plus": 5072.0, + "lt50": 6280.0, + "total": 5201.0 + }, + "34": { + "50plus": 4675.0, + "lt50": 6836.0, + "total": 5052.0 + }, + "35": { + "50plus": 4317.0, + "lt50": 5360.0, + "total": 4461.0, + "unreliable": [ + "lt50" + ] + }, + "36": { + "50plus": 4604.0, + "lt50": 6460.0, + "total": 4842.0 + }, + "37": { + "50plus": 5068.0, + "lt50": 4148.0, + "total": 4973.0 + }, + "38": { + "50plus": 4921.0, + "lt50": 3347.0, + "total": 4640.0 + }, + "39": { + "50plus": 4081.0, + "lt50": 5839.0, + "total": 4226.0 + }, + "40": { + "50plus": 4768.0, + "lt50": 6016.0, + "total": 4926.0 + }, + "41": { + "50plus": 4743.0, + "lt50": 5794.0, + "total": 4915.0 + }, + "42": { + "50plus": 3846.0, + "lt50": 3916.0, + "total": 3855.0 + }, + "44": { + "50plus": 4813.0, + "lt50": 3981.0, + "total": 4706.0 + }, + "45": { + "50plus": 4919.0, + "lt50": null, + "total": 4879.0 + }, + "46": { + "50plus": 4700.0, + "lt50": null, + "total": 4620.0 + }, + "47": { + "50plus": 4023.0, + "lt50": 5877.0, + "total": 4236.0 + }, + "48": { + "50plus": 4964.0, + "lt50": 8450.0, + "total": 5237.0 + }, + "49": { + "50plus": 3562.0, + "lt50": 4771.0, + "total": 3719.0 + }, + "50": { + "50plus": 4787.0, + "lt50": 7994.0, + "total": 5166.0 + }, + "51": { + "50plus": 4821.0, + "lt50": 6957.0, + "total": 4979.0 + }, + "53": { + "50plus": 4113.0, + "lt50": 5301.0, + "total": 4304.0 + }, + "54": { + "50plus": 3938.0, + "lt50": 5036.0, + "total": 4013.0 + }, + "55": { + "50plus": 4680.0, + "lt50": 5010.0, + "total": 4724.0 + }, + "56": { + "50plus": 4055.0, + "lt50": 6009.0, + "total": 4612.0 + }, + "US": { + "50plus": 4635.0, + "lt50": 5809.0, + "total": 4776.0 + } + }, + "text_line": 16849, + "title": "Table II.E.2 Average total employee contribution (in dollars) per enrolled employee for employee-plus-one coverage at private-sector" + }, + "premium": { + "rows": { + "01": { + "50plus": 16418.0, + "lt50": 14957.0, + "total": 16224.0 + }, + "02": { + "50plus": 20802.0, + "lt50": 20379.0, + "total": 20764.0 + }, + "04": { + "50plus": 15210.0, + "lt50": 11974.0, + "total": 14664.0 + }, + "05": { + "50plus": 15948.0, + "lt50": 14031.0, + "total": 15632.0 + }, + "06": { + "50plus": 18968.0, + "lt50": 17950.0, + "total": 18827.0 + }, + "08": { + "50plus": 17168.0, + "lt50": 19954.0, + "total": 17447.0 + }, + "09": { + "50plus": 19797.0, + "lt50": 16920.0, + "total": 19523.0 + }, + "10": { + "50plus": 18172.0, + "lt50": 16742.0, + "total": 17997.0 + }, + "11": { + "50plus": 19577.0, + "lt50": 19173.0, + "total": 19518.0 + }, + "12": { + "50plus": 17458.0, + "lt50": 17221.0, + "total": 17433.0 + }, + "13": { + "50plus": 17748.0, + "lt50": 22802.0, + "total": 18227.0 + }, + "15": { + "50plus": 17926.0, + "lt50": 15739.0, + "total": 17453.0 + }, + "16": { + "50plus": 17903.0, + "lt50": 16185.0, + "total": 17652.0 + }, + "17": { + "50plus": 19179.0, + "lt50": 20228.0, + "total": 19330.0 + }, + "18": { + "50plus": 15875.0, + "lt50": 15355.0, + "total": 15825.0 + }, + "19": { + "50plus": 17103.0, + "lt50": 14736.0, + "total": 16782.0 + }, + "20": { + "50plus": 17861.0, + "lt50": 15480.0, + "total": 17574.0 + }, + "21": { + "50plus": 15109.0, + "lt50": 18513.0, + "total": 15348.0 + }, + "22": { + "50plus": 17872.0, + "lt50": 18432.0, + "total": 17941.0 + }, + "23": { + "50plus": 18969.0, + "lt50": 18479.0, + "total": 18908.0 + }, + "24": { + "50plus": 18615.0, + "lt50": 16466.0, + "total": 18373.0 + }, + "25": { + "50plus": 20135.0, + "lt50": 18465.0, + "total": 19960.0 + }, + "26": { + "50plus": 16469.0, + "lt50": 17546.0, + "total": 16678.0 + }, + "27": { + "50plus": 18913.0, + "lt50": 16078.0, + "total": 18599.0 + }, + "28": { + "50plus": 14870.0, + "lt50": 14645.0, + "total": 14841.0 + }, + "29": { + "50plus": 17125.0, + "lt50": 16241.0, + "total": 17045.0 + }, + "30": { + "50plus": 15634.0, + "lt50": 16382.0, + "total": 15842.0 + }, + "31": { + "50plus": 17402.0, + "lt50": 15704.0, + "total": 17232.0 + }, + "32": { + "50plus": 14982.0, + "lt50": 13611.0, + "total": 14858.0 + }, + "33": { + "50plus": 20224.0, + "lt50": 17855.0, + "total": 19972.0 + }, + "34": { + "50plus": 19631.0, + "lt50": 19176.0, + "total": 19551.0 + }, + "35": { + "50plus": 19059.0, + "lt50": 21911.0, + "total": 19454.0 + }, + "36": { + "50plus": 20376.0, + "lt50": 23553.0, + "total": 20784.0 + }, + "37": { + "50plus": 16396.0, + "lt50": 12131.0, + "total": 15958.0 + }, + "38": { + "50plus": 18518.0, + "lt50": 13977.0, + "total": 17707.0 + }, + "39": { + "50plus": 18113.0, + "lt50": 16253.0, + "total": 17960.0 + }, + "40": { + "50plus": 16288.0, + "lt50": 14048.0, + "total": 16005.0 + }, + "41": { + "50plus": 18239.0, + "lt50": 16815.0, + "total": 18006.0 + }, + "42": { + "50plus": 19736.0, + "lt50": 17599.0, + "total": 19455.0 + }, + "44": { + "50plus": 20372.0, + "lt50": 16114.0, + "total": 19825.0 + }, + "45": { + "50plus": 16508.0, + "lt50": 14947.0, + "total": 16317.0 + }, + "46": { + "50plus": 18264.0, + "lt50": 14750.0, + "total": 17990.0 + }, + "47": { + "50plus": 16796.0, + "lt50": 16341.0, + "total": 16743.0 + }, + "48": { + "50plus": 16510.0, + "lt50": 18357.0, + "total": 16654.0 + }, + "49": { + "50plus": 16949.0, + "lt50": 14209.0, + "total": 16593.0 + }, + "50": { + "50plus": 18840.0, + "lt50": 21527.0, + "total": 19157.0 + }, + "51": { + "50plus": 17533.0, + "lt50": 16454.0, + "total": 17453.0 + }, + "53": { + "50plus": 18121.0, + "lt50": 17787.0, + "total": 18067.0 + }, + "54": { + "50plus": 18824.0, + "lt50": 13947.0, + "total": 18492.0 + }, + "55": { + "50plus": 19345.0, + "lt50": 19165.0, + "total": 19321.0 + }, + "56": { + "50plus": 19248.0, + "lt50": 16466.0, + "total": 18455.0 + }, + "US": { + "50plus": 17932.0, + "lt50": 17678.0, + "total": 17901.0 + } + }, + "text_line": 16351, + "title": "Table II.E.1 Average total employee-plus-one premium (in dollars) per enrolled employee at private-sector establishments that offer" + } + }, + "family": { + "employee_contribution": { + "rows": { + "01": { + "50plus": 7797.0, + "lt50": 5592.0, + "total": 7540.0 + }, + "02": { + "50plus": 6272.0, + "lt50": null, + "total": 6171.0 + }, + "04": { + "50plus": 8624.0, + "lt50": 7732.0, + "total": 8493.0 + }, + "05": { + "50plus": 8205.0, + "lt50": 5480.0, + "total": 7824.0 + }, + "06": { + "50plus": 7972.0, + "lt50": 7166.0, + "total": 7847.0 + }, + "08": { + "50plus": 6659.0, + "lt50": 7191.0, + "total": 6733.0 + }, + "09": { + "50plus": 4614.0, + "lt50": 6788.0, + "total": 4979.0, + "unreliable": [ + "lt50" + ] + }, + "10": { + "50plus": 7184.0, + "lt50": null, + "total": 6993.0 + }, + "11": { + "50plus": 8191.0, + "lt50": 7114.0, + "total": 8028.0 + }, + "12": { + "50plus": 7676.0, + "lt50": 10263.0, + "total": 7976.0 + }, + "13": { + "50plus": 7379.0, + "lt50": 5677.0, + "total": 7224.0 + }, + "15": { + "50plus": 5310.0, + "lt50": 6819.0, + "total": 5531.0 + }, + "16": { + "50plus": 6506.0, + "lt50": 6135.0, + "total": 6439.0 + }, + "17": { + "50plus": 7911.0, + "lt50": 7377.0, + "total": 7838.0 + }, + "18": { + "50plus": 6808.0, + "lt50": 5322.0, + "total": 6650.0, + "unreliable": [ + "lt50" + ] + }, + "19": { + "50plus": 7102.0, + "lt50": 10963.0, + "total": 7533.0 + }, + "20": { + "50plus": 7908.0, + "lt50": 9353.0, + "total": 8222.0 + }, + "21": { + "50plus": 5858.0, + "lt50": 7769.0, + "total": 5982.0 + }, + "22": { + "50plus": 7191.0, + "lt50": 11197.0, + "total": 7808.0 + }, + "23": { + "50plus": 6581.0, + "lt50": 10518.0, + "total": 7079.0 + }, + "24": { + "50plus": 6107.0, + "lt50": 10129.0, + "total": 6502.0 + }, + "25": { + "50plus": 7392.0, + "lt50": 7943.0, + "total": 7501.0 + }, + "26": { + "50plus": 6199.0, + "lt50": 7727.0, + "total": 6437.0 + }, + "27": { + "50plus": 6622.0, + "lt50": 9801.0, + "total": 7013.0 + }, + "28": { + "50plus": 6021.0, + "lt50": 8694.0, + "total": 6228.0 + }, + "29": { + "50plus": 6662.0, + "lt50": 7909.0, + "total": 6797.0 + }, + "30": { + "50plus": 5282.0, + "lt50": 4365.0, + "total": 4967.0, + "unreliable": [ + "lt50" + ] + }, + "31": { + "50plus": 6641.0, + "lt50": 4775.0, + "total": 6369.0 + }, + "32": { + "50plus": 7953.0, + "lt50": 7528.0, + "total": 7902.0 + }, + "33": { + "50plus": 7926.0, + "lt50": 5703.0, + "total": 7434.0, + "unreliable": [ + "lt50" + ] + }, + "34": { + "50plus": 6745.0, + "lt50": 9849.0, + "total": 7053.0 + }, + "35": { + "50plus": 6586.0, + "lt50": null, + "total": 6761.0 + }, + "36": { + "50plus": 6596.0, + "lt50": 9527.0, + "total": 7075.0 + }, + "37": { + "50plus": 7894.0, + "lt50": 10336.0, + "total": 8073.0 + }, + "38": { + "50plus": 5632.0, + "lt50": 5409.0, + "total": 5586.0 + }, + "39": { + "50plus": 6828.0, + "lt50": 4392.0, + "total": 6453.0, + "unreliable": [ + "lt50" + ] + }, + "40": { + "50plus": 6820.0, + "lt50": 7568.0, + "total": 6955.0 + }, + "41": { + "50plus": 9550.0, + "lt50": 8246.0, + "total": 9374.0 + }, + "42": { + "50plus": 6936.0, + "lt50": 5659.0, + "total": 6720.0 + }, + "44": { + "50plus": 7002.0, + "lt50": 7194.0, + "total": 7028.0 + }, + "45": { + "50plus": 8307.0, + "lt50": null, + "total": 8269.0 + }, + "46": { + "50plus": 6983.0, + "lt50": 6288.0, + "total": 6892.0 + }, + "47": { + "50plus": 5680.0, + "lt50": 9127.0, + "total": 5972.0 + }, + "48": { + "50plus": 7571.0, + "lt50": 11247.0, + "total": 7913.0 + }, + "49": { + "50plus": 5625.0, + "lt50": 5199.0, + "total": 5567.0 + }, + "50": { + "50plus": 7324.0, + "lt50": 11701.0, + "total": 7870.0 + }, + "51": { + "50plus": 9012.0, + "lt50": 6839.0, + "total": 8692.0 + }, + "53": { + "50plus": 8501.0, + "lt50": 5989.0, + "total": 8222.0 + }, + "54": { + "50plus": 4966.0, + "lt50": null, + "total": 5129.0 + }, + "55": { + "50plus": 6698.0, + "lt50": 3464.0, + "total": 6148.0 + }, + "56": { + "50plus": 6666.0, + "lt50": 7410.0, + "total": 6830.0 + }, + "US": { + "50plus": 7246.0, + "lt50": 7760.0, + "total": 7314.0 + } + }, + "text_line": 13495, + "title": "Table II.D.2 Average total employee contribution (in dollars) per enrolled employee for family coverage at private-sector establishments" + }, + "premium": { + "rows": { + "01": { + "50plus": 24093.0, + "lt50": 19467.0, + "total": 23554.0 + }, + "02": { + "50plus": 29538.0, + "lt50": 39824.0, + "total": 30132.0 + }, + "04": { + "50plus": 23190.0, + "lt50": 22869.0, + "total": 23143.0 + }, + "05": { + "50plus": 23777.0, + "lt50": 17908.0, + "total": 22956.0 + }, + "06": { + "50plus": 27727.0, + "lt50": 23389.0, + "total": 27057.0 + }, + "08": { + "50plus": 25668.0, + "lt50": 21275.0, + "total": 25055.0 + }, + "09": { + "50plus": 29669.0, + "lt50": 23892.0, + "total": 28700.0 + }, + "10": { + "50plus": 26070.0, + "lt50": 22354.0, + "total": 25600.0 + }, + "11": { + "50plus": 29886.0, + "lt50": 26474.0, + "total": 29371.0 + }, + "12": { + "50plus": 25454.0, + "lt50": 24957.0, + "total": 25396.0 + }, + "13": { + "50plus": 26308.0, + "lt50": 27747.0, + "total": 26439.0 + }, + "15": { + "50plus": 23278.0, + "lt50": 25498.0, + "total": 23603.0 + }, + "16": { + "50plus": 26821.0, + "lt50": 22091.0, + "total": 25957.0 + }, + "17": { + "50plus": 28288.0, + "lt50": 27996.0, + "total": 28248.0 + }, + "18": { + "50plus": 23868.0, + "lt50": 20362.0, + "total": 23497.0 + }, + "19": { + "50plus": 25195.0, + "lt50": 25071.0, + "total": 25181.0 + }, + "20": { + "50plus": 27038.0, + "lt50": 19915.0, + "total": 25493.0 + }, + "21": { + "50plus": 24775.0, + "lt50": 20839.0, + "total": 24520.0 + }, + "22": { + "50plus": 26099.0, + "lt50": 23163.0, + "total": 25647.0 + }, + "23": { + "50plus": 26617.0, + "lt50": 22577.0, + "total": 26106.0 + }, + "24": { + "50plus": 24670.0, + "lt50": 25156.0, + "total": 24717.0 + }, + "25": { + "50plus": 30238.0, + "lt50": 27129.0, + "total": 29619.0 + }, + "26": { + "50plus": 23913.0, + "lt50": 23191.0, + "total": 23800.0 + }, + "27": { + "50plus": 26446.0, + "lt50": 22393.0, + "total": 25947.0 + }, + "28": { + "50plus": 22249.0, + "lt50": 18158.0, + "total": 21932.0 + }, + "29": { + "50plus": 25723.0, + "lt50": 23850.0, + "total": 25520.0 + }, + "30": { + "50plus": 22878.0, + "lt50": 20292.0, + "total": 21989.0 + }, + "31": { + "50plus": 26754.0, + "lt50": 27269.0, + "total": 26830.0 + }, + "32": { + "50plus": 23408.0, + "lt50": 20601.0, + "total": 23075.0 + }, + "33": { + "50plus": 29510.0, + "lt50": 23407.0, + "total": 28161.0 + }, + "34": { + "50plus": 30043.0, + "lt50": 26238.0, + "total": 29666.0 + }, + "35": { + "50plus": 26403.0, + "lt50": 27361.0, + "total": 26482.0 + }, + "36": { + "50plus": 29635.0, + "lt50": 30351.0, + "total": 29752.0 + }, + "37": { + "50plus": 25467.0, + "lt50": 23343.0, + "total": 25311.0 + }, + "38": { + "50plus": 25153.0, + "lt50": 23944.0, + "total": 24903.0 + }, + "39": { + "50plus": 25377.0, + "lt50": 20726.0, + "total": 24661.0 + }, + "40": { + "50plus": 24628.0, + "lt50": 23828.0, + "total": 24483.0 + }, + "41": { + "50plus": 28022.0, + "lt50": 21815.0, + "total": 27182.0 + }, + "42": { + "50plus": 27077.0, + "lt50": 24389.0, + "total": 26623.0 + }, + "44": { + "50plus": 28020.0, + "lt50": 23044.0, + "total": 27361.0 + }, + "45": { + "50plus": 25209.0, + "lt50": 23531.0, + "total": 25046.0 + }, + "46": { + "50plus": 26661.0, + "lt50": 22476.0, + "total": 26110.0 + }, + "47": { + "50plus": 25166.0, + "lt50": 24654.0, + "total": 25123.0 + }, + "48": { + "50plus": 25371.0, + "lt50": 21949.0, + "total": 25053.0 + }, + "49": { + "50plus": 25736.0, + "lt50": 21266.0, + "total": 25122.0 + }, + "50": { + "50plus": 25016.0, + "lt50": 28494.0, + "total": 25450.0 + }, + "51": { + "50plus": 28538.0, + "lt50": 19835.0, + "total": 27257.0 + }, + "53": { + "50plus": 28199.0, + "lt50": 22978.0, + "total": 27618.0 + }, + "54": { + "50plus": 26213.0, + "lt50": 24948.0, + "total": 26094.0 + }, + "55": { + "50plus": 27093.0, + "lt50": 28362.0, + "total": 27309.0 + }, + "56": { + "50plus": 27668.0, + "lt50": 25028.0, + "total": 27084.0 + }, + "US": { + "50plus": 26602.0, + "lt50": 24176.0, + "total": 26281.0 + } + }, + "text_line": 12499, + "title": "Table II.D.1 Average total family premium (in dollars) per enrolled employee at private-sector establishments that offer health" + } + }, + "single": { + "employee_contribution": { + "rows": { + "01": { + "50plus": 2005.0, + "lt50": 2336.0, + "total": 2048.0 + }, + "02": { + "50plus": 1945.0, + "lt50": 1866.0, + "total": 1931.0 + }, + "04": { + "50plus": 1722.0, + "lt50": 2503.0, + "total": 1837.0, + "unreliable": [ + "lt50" + ] + }, + "05": { + "50plus": 1859.0, + "lt50": 1404.0, + "total": 1795.0 + }, + "06": { + "50plus": 1499.0, + "lt50": 1602.0, + "total": 1515.0 + }, + "08": { + "50plus": 1685.0, + "lt50": 1295.0, + "total": 1618.0 + }, + "09": { + "50plus": 1755.0, + "lt50": 1843.0, + "total": 1771.0 + }, + "10": { + "50plus": 2179.0, + "lt50": 1806.0, + "total": 2120.0, + "unreliable": [ + "lt50" + ] + }, + "11": { + "50plus": 1885.0, + "lt50": 1777.0, + "total": 1863.0 + }, + "12": { + "50plus": 1825.0, + "lt50": 2500.0, + "total": 1913.0 + }, + "13": { + "50plus": 1710.0, + "lt50": 1994.0, + "total": 1748.0 + }, + "15": { + "50plus": 1040.0, + "lt50": 404.0, + "total": 857.0, + "unreliable": [ + "lt50" + ] + }, + "16": { + "50plus": 1733.0, + "lt50": 1477.0, + "total": 1686.0 + }, + "17": { + "50plus": 1820.0, + "lt50": 2112.0, + "total": 1858.0 + }, + "18": { + "50plus": 2158.0, + "lt50": 1920.0, + "total": 2137.0 + }, + "19": { + "50plus": 2003.0, + "lt50": 2150.0, + "total": 2025.0 + }, + "20": { + "50plus": 1796.0, + "lt50": 1724.0, + "total": 1784.0 + }, + "21": { + "50plus": 1884.0, + "lt50": 2306.0, + "total": 1947.0, + "unreliable": [ + "lt50" + ] + }, + "22": { + "50plus": 2123.0, + "lt50": 2727.0, + "total": 2259.0 + }, + "23": { + "50plus": 1846.0, + "lt50": 1963.0, + "total": 1869.0 + }, + "24": { + "50plus": 2193.0, + "lt50": 1586.0, + "total": 2065.0 + }, + "25": { + "50plus": 1930.0, + "lt50": 2497.0, + "total": 2038.0 + }, + "26": { + "50plus": 1945.0, + "lt50": 2075.0, + "total": 1970.0 + }, + "27": { + "50plus": 1770.0, + "lt50": 2661.0, + "total": 1939.0 + }, + "28": { + "50plus": 1743.0, + "lt50": 1566.0, + "total": 1716.0 + }, + "29": { + "50plus": 1645.0, + "lt50": 1671.0, + "total": 1649.0 + }, + "30": { + "50plus": 1343.0, + "lt50": 1799.0, + "total": 1493.0 + }, + "31": { + "50plus": 1602.0, + "lt50": 2206.0, + "total": 1722.0 + }, + "32": { + "50plus": 1586.0, + "lt50": 1422.0, + "total": 1563.0 + }, + "33": { + "50plus": 2335.0, + "lt50": 2410.0, + "total": 2350.0 + }, + "34": { + "50plus": 2088.0, + "lt50": 2698.0, + "total": 2223.0 + }, + "35": { + "50plus": 1682.0, + "lt50": 2171.0, + "total": 1767.0, + "unreliable": [ + "lt50" + ] + }, + "36": { + "50plus": 2119.0, + "lt50": 2885.0, + "total": 2231.0 + }, + "37": { + "50plus": 2013.0, + "lt50": 2012.0, + "total": 2012.0 + }, + "38": { + "50plus": 1737.0, + "lt50": 1179.0, + "total": 1579.0 + }, + "39": { + "50plus": 1985.0, + "lt50": 2206.0, + "total": 2016.0 + }, + "40": { + "50plus": 1517.0, + "lt50": 1750.0, + "total": 1558.0 + }, + "41": { + "50plus": 1514.0, + "lt50": 1448.0, + "total": 1499.0 + }, + "42": { + "50plus": 1645.0, + "lt50": 1774.0, + "total": 1669.0 + }, + "44": { + "50plus": 2082.0, + "lt50": 1902.0, + "total": 2046.0 + }, + "45": { + "50plus": 1804.0, + "lt50": 1881.0, + "total": 1814.0 + }, + "46": { + "50plus": 1665.0, + "lt50": 2002.0, + "total": 1730.0 + }, + "47": { + "50plus": 1743.0, + "lt50": 986.0, + "total": 1645.0 + }, + "48": { + "50plus": 1646.0, + "lt50": 1583.0, + "total": 1638.0 + }, + "49": { + "50plus": 1621.0, + "lt50": 1328.0, + "total": 1591.0 + }, + "50": { + "50plus": 2022.0, + "lt50": 2622.0, + "total": 2187.0 + }, + "51": { + "50plus": 2120.0, + "lt50": 1763.0, + "total": 2054.0 + }, + "53": { + "50plus": 1462.0, + "lt50": 1088.0, + "total": 1381.0 + }, + "54": { + "50plus": 2118.0, + "lt50": 1388.0, + "total": 2012.0, + "unreliable": [ + "lt50" + ] + }, + "55": { + "50plus": 2102.0, + "lt50": 1809.0, + "total": 2052.0 + }, + "56": { + "50plus": 1620.0, + "lt50": 1258.0, + "total": 1497.0, + "unreliable": [ + "lt50" + ] + }, + "US": { + "50plus": 1796.0, + "lt50": 1924.0, + "total": 1817.0 + } + }, + "text_line": 9647, + "title": "Table II.C.2 Average total employee contribution (in dollars) per enrolled employee for single coverage at private-sector establishments" + }, + "premium": { + "rows": { + "01": { + "50plus": 8216.0, + "lt50": 7854.0, + "total": 8169.0 + }, + "02": { + "50plus": 10536.0, + "lt50": 11082.0, + "total": 10630.0 + }, + "04": { + "50plus": 7578.0, + "lt50": 8640.0, + "total": 7734.0 + }, + "05": { + "50plus": 7823.0, + "lt50": 7691.0, + "total": 7804.0 + }, + "06": { + "50plus": 9164.0, + "lt50": 9149.0, + "total": 9162.0 + }, + "08": { + "50plus": 8977.0, + "lt50": 8039.0, + "total": 8816.0 + }, + "09": { + "50plus": 10364.0, + "lt50": 8243.0, + "total": 9987.0 + }, + "10": { + "50plus": 9247.0, + "lt50": 10589.0, + "total": 9459.0 + }, + "11": { + "50plus": 9895.0, + "lt50": 9252.0, + "total": 9762.0 + }, + "12": { + "50plus": 8795.0, + "lt50": 9742.0, + "total": 8919.0 + }, + "13": { + "50plus": 8768.0, + "lt50": 9801.0, + "total": 8907.0 + }, + "15": { + "50plus": 8250.0, + "lt50": 8372.0, + "total": 8285.0 + }, + "16": { + "50plus": 8760.0, + "lt50": 8815.0, + "total": 8770.0 + }, + "17": { + "50plus": 9703.0, + "lt50": 9865.0, + "total": 9724.0 + }, + "18": { + "50plus": 8455.0, + "lt50": 7330.0, + "total": 8355.0 + }, + "19": { + "50plus": 8629.0, + "lt50": 8572.0, + "total": 8620.0 + }, + "20": { + "50plus": 8874.0, + "lt50": 7572.0, + "total": 8657.0 + }, + "21": { + "50plus": 8810.0, + "lt50": 9644.0, + "total": 8936.0 + }, + "22": { + "50plus": 8624.0, + "lt50": 9429.0, + "total": 8806.0 + }, + "23": { + "50plus": 9983.0, + "lt50": 9062.0, + "total": 9803.0 + }, + "24": { + "50plus": 9061.0, + "lt50": 7628.0, + "total": 8760.0 + }, + "25": { + "50plus": 10548.0, + "lt50": 9965.0, + "total": 10436.0 + }, + "26": { + "50plus": 8193.0, + "lt50": 8384.0, + "total": 8230.0 + }, + "27": { + "50plus": 8644.0, + "lt50": 8336.0, + "total": 8586.0 + }, + "28": { + "50plus": 7637.0, + "lt50": 7477.0, + "total": 7612.0 + }, + "29": { + "50plus": 8039.0, + "lt50": 8370.0, + "total": 8084.0 + }, + "30": { + "50plus": 8826.0, + "lt50": 9049.0, + "total": 8899.0 + }, + "31": { + "50plus": 8822.0, + "lt50": 9253.0, + "total": 8907.0 + }, + "32": { + "50plus": 8213.0, + "lt50": 8232.0, + "total": 8215.0 + }, + "33": { + "50plus": 9768.0, + "lt50": 8505.0, + "total": 9502.0 + }, + "34": { + "50plus": 9552.0, + "lt50": 10415.0, + "total": 9743.0 + }, + "35": { + "50plus": 9445.0, + "lt50": 9459.0, + "total": 9447.0 + }, + "36": { + "50plus": 10760.0, + "lt50": 11804.0, + "total": 10913.0 + }, + "37": { + "50plus": 8728.0, + "lt50": 7872.0, + "total": 8612.0 + }, + "38": { + "50plus": 9349.0, + "lt50": 9238.0, + "total": 9318.0 + }, + "39": { + "50plus": 8872.0, + "lt50": 8379.0, + "total": 8804.0 + }, + "40": { + "50plus": 8101.0, + "lt50": 8231.0, + "total": 8124.0 + }, + "41": { + "50plus": 9407.0, + "lt50": 8594.0, + "total": 9225.0 + }, + "42": { + "50plus": 9619.0, + "lt50": 8554.0, + "total": 9421.0 + }, + "44": { + "50plus": 9369.0, + "lt50": 9465.0, + "total": 9388.0 + }, + "45": { + "50plus": 8479.0, + "lt50": 9020.0, + "total": 8552.0 + }, + "46": { + "50plus": 9225.0, + "lt50": 7990.0, + "total": 8985.0 + }, + "47": { + "50plus": 8712.0, + "lt50": 8365.0, + "total": 8667.0 + }, + "48": { + "50plus": 8514.0, + "lt50": 8737.0, + "total": 8543.0 + }, + "49": { + "50plus": 8533.0, + "lt50": 7702.0, + "total": 8448.0 + }, + "50": { + "50plus": 9452.0, + "lt50": 11460.0, + "total": 10004.0 + }, + "51": { + "50plus": 9157.0, + "lt50": 8141.0, + "total": 8970.0 + }, + "53": { + "50plus": 8592.0, + "lt50": 9146.0, + "total": 8712.0 + }, + "54": { + "50plus": 8897.0, + "lt50": 8763.0, + "total": 8878.0 + }, + "55": { + "50plus": 9618.0, + "lt50": 9260.0, + "total": 9557.0 + }, + "56": { + "50plus": 9155.0, + "lt50": 8691.0, + "total": 8997.0 + }, + "US": { + "50plus": 9024.0, + "lt50": 9034.0, + "total": 9025.0 + } + }, + "text_line": 8651, + "title": "Table II.C.1 Average total single premium (in dollars) per enrolled employee at private-sector establishments that offer health" + } + } + }, + "public_division_2024": { + "employee_plus_one": { + "employee_contribution": { + "rows": { + "East North Central": { + "all_state_and_local": 3617.0, + "state": 3387.0 + }, + "East South Central": { + "all_state_and_local": 3074.0, + "state": 2391.0 + }, + "Middle Atlantic": { + "all_state_and_local": 3294.0, + "state": 3453.0 + }, + "Mountain": { + "all_state_and_local": 3106.0, + "state": 1849.0 + }, + "New England": { + "all_state_and_local": 4414.0, + "state": 3936.0 + }, + "Pacific": { + "all_state_and_local": 2213.0, + "state": 2882.0 + }, + "South Atlantic": { + "all_state_and_local": 4094.0, + "state": 3603.0 + }, + "US": { + "all_state_and_local": 3426.0, + "state": 2973.0 + }, + "West North Central": { + "all_state_and_local": 3343.0, + "state": 1901.0 + }, + "West South Central": { + "all_state_and_local": 4215.0, + "state": 3218.0 + } + }, + "text_line": 3490, + "title": "Table III.E.2 Average total employee contribution (in dollars) per enrolled employee for employee-plus-one coverage through State and" + }, + "premium": { + "rows": { + "East North Central": { + "all_state_and_local": 19183.0, + "state": 18187.0 + }, + "East South Central": { + "all_state_and_local": 14689.0, + "state": 13091.0 + }, + "Middle Atlantic": { + "all_state_and_local": 22282.0, + "state": 19103.0 + }, + "Mountain": { + "all_state_and_local": 16821.0, + "state": 15235.0 + }, + "New England": { + "all_state_and_local": 24306.0, + "state": 23854.0 + }, + "Pacific": { + "all_state_and_local": 19694.0, + "state": 20386.0 + }, + "South Atlantic": { + "all_state_and_local": 17468.0, + "state": 17176.0 + }, + "US": { + "all_state_and_local": 17939.0, + "state": 17352.0 + }, + "West North Central": { + "all_state_and_local": 15648.0, + "state": 14657.0 + }, + "West South Central": { + "all_state_and_local": 14600.0, + "state": 14585.0 + } + }, + "text_line": 3322, + "title": "Table III.E.1 Average total employee-plus-one premium (in dollars) per employee enrolled through State and local government jobs by" + } + }, + "family": { + "employee_contribution": { + "rows": { + "East North Central": { + "all_state_and_local": 4408.0, + "state": 4357.0 + }, + "East South Central": { + "all_state_and_local": 5192.0, + "state": 4598.0 + }, + "Middle Atlantic": { + "all_state_and_local": 4104.0, + "state": 5230.0 + }, + "Mountain": { + "all_state_and_local": 4587.0, + "state": 3518.0 + }, + "New England": { + "all_state_and_local": 7396.0, + "state": 6329.0 + }, + "Pacific": { + "all_state_and_local": 2948.0, + "state": 3226.0 + }, + "South Atlantic": { + "all_state_and_local": 5559.0, + "state": 4133.0 + }, + "US": { + "all_state_and_local": 4763.0, + "state": 4407.0 + }, + "West North Central": { + "all_state_and_local": 4909.0, + "state": 3565.0 + }, + "West South Central": { + "all_state_and_local": 8335.0, + "state": 7268.0 + } + }, + "text_line": 2740, + "title": "Table III.D.2 Average total employee contribution (in dollars) per employee enrolled for family coverage through State and local" + }, + "premium": { + "rows": { + "East North Central": { + "all_state_and_local": 26002.0, + "state": 26404.0 + }, + "East South Central": { + "all_state_and_local": 21068.0, + "state": 19138.0 + }, + "Middle Atlantic": { + "all_state_and_local": 29797.0, + "state": 27676.0 + }, + "Mountain": { + "all_state_and_local": 24041.0, + "state": 22784.0 + }, + "New England": { + "all_state_and_local": 29954.0, + "state": 28114.0 + }, + "Pacific": { + "all_state_and_local": 25909.0, + "state": 27689.0 + }, + "South Atlantic": { + "all_state_and_local": 24204.0, + "state": 23159.0 + }, + "US": { + "all_state_and_local": 26010.0, + "state": 25674.0 + }, + "West North Central": { + "all_state_and_local": 25349.0, + "state": 25951.0 + }, + "West South Central": { + "all_state_and_local": 22682.0, + "state": 22947.0 + } + }, + "text_line": 2572, + "title": "Table III.D.1 Average total family premium (in dollars) per employee enrolled through State and local government jobs by government" + } + }, + "single": { + "employee_contribution": { + "rows": { + "East North Central": { + "all_state_and_local": 1236.0, + "state": 1346.0 + }, + "East South Central": { + "all_state_and_local": 1004.0, + "state": 825.0 + }, + "Middle Atlantic": { + "all_state_and_local": 1314.0, + "state": 1184.0 + }, + "Mountain": { + "all_state_and_local": 781.0, + "state": 935.0 + }, + "New England": { + "all_state_and_local": 2563.0, + "state": 2076.0 + }, + "Pacific": { + "all_state_and_local": 976.0, + "state": 981.0 + }, + "South Atlantic": { + "all_state_and_local": 939.0, + "state": 911.0 + }, + "US": { + "all_state_and_local": 1057.0, + "state": 1024.0 + }, + "West North Central": { + "all_state_and_local": 614.0, + "state": 695.0 + }, + "West South Central": { + "all_state_and_local": 1023.0, + "state": 796.0 + } + }, + "text_line": 1996, + "title": "Table III.C.2 Average total employee contribution (in dollars) per employee enrolled for single coverage through State and local" + }, + "premium": { + "rows": { + "East North Central": { + "all_state_and_local": 9975.0, + "state": 9324.0 + }, + "East South Central": { + "all_state_and_local": 8307.0, + "state": 7555.0 + }, + "Middle Atlantic": { + "all_state_and_local": 11049.0, + "state": 9680.0 + }, + "Mountain": { + "all_state_and_local": 9066.0, + "state": 9452.0 + }, + "New England": { + "all_state_and_local": 11614.0, + "state": 10757.0 + }, + "Pacific": { + "all_state_and_local": 10997.0, + "state": 10633.0 + }, + "South Atlantic": { + "all_state_and_local": 9672.0, + "state": 8891.0 + }, + "US": { + "all_state_and_local": 9687.0, + "state": 9342.0 + }, + "West North Central": { + "all_state_and_local": 8757.0, + "state": 9060.0 + }, + "West South Central": { + "all_state_and_local": 8302.0, + "state": 8564.0 + } + }, + "text_line": 1828, + "title": "Table III.C.1 Average total single premium (in dollars) per employee enrolled through State and local government jobs by government" + } + } + }, + "schema_version": 1, + "sources": { + "private_national_2025": { + "description": "MEPS-IC 2025 Series I (private sector, national): Table I.A.2 (percent of establishments that offer health insurance) and Table I.A.2.j (percent of establishments that offer pretax employee contributions to health insurance), by firm size", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3", + "size_bytes": 9210366, + "survey_year": 2025, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_1/2025/ic25_ia_g.pdf" + }, + "private_state_2024": { + "description": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", + "size_bytes": 10687163, + "survey_year": 2024, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf" + }, + "private_state_2025": { + "description": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee) and II.C/D/E.2 (average employee contribution)", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", + "size_bytes": 12530005, + "survey_year": 2025, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf" + }, + "public_division_2024": { + "description": "MEPS-IC 2024 Series III (State and local government, by government type and size and census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released).", + "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", + "sha256": "462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79", + "size_bytes": 1765717, + "survey_year": 2024, + "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_3/2024/ic24_iiia_g.pdf" + } + }, + "state_census_division": { + "01": "East South Central", + "02": "Pacific", + "04": "Mountain", + "05": "West South Central", + "06": "Pacific", + "08": "Mountain", + "09": "New England", + "10": "South Atlantic", + "11": "South Atlantic", + "12": "South Atlantic", + "13": "South Atlantic", + "15": "Pacific", + "16": "Mountain", + "17": "East North Central", + "18": "East North Central", + "19": "West North Central", + "20": "West North Central", + "21": "East South Central", + "22": "West South Central", + "23": "New England", + "24": "South Atlantic", + "25": "New England", + "26": "East North Central", + "27": "West North Central", + "28": "East South Central", + "29": "West North Central", + "30": "Mountain", + "31": "West North Central", + "32": "Mountain", + "33": "New England", + "34": "Middle Atlantic", + "35": "Mountain", + "36": "Middle Atlantic", + "37": "South Atlantic", + "38": "West North Central", + "39": "East North Central", + "40": "West South Central", + "41": "Pacific", + "42": "Middle Atlantic", + "44": "New England", + "45": "South Atlantic", + "46": "West North Central", + "47": "East South Central", + "48": "West South Central", + "49": "Mountain", + "50": "New England", + "51": "South Atlantic", + "53": "Pacific", + "54": "South Atlantic", + "55": "East North Central", + "56": "Mountain" + }, + "tiers": [ + "single", + "employee_plus_one", + "family" + ] +} diff --git a/tools/build_us_meps_ic_esi_cells.py b/tools/build_us_meps_ic_esi_cells.py new file mode 100644 index 0000000000..18a6819659 --- /dev/null +++ b/tools/build_us_meps_ic_esi_cells.py @@ -0,0 +1,548 @@ +"""Regenerate the packaged MEPS-IC employer-premium cell table (microcosm#454). + +The ``meps_esi_premiums`` source stage assigns each employed ESI policyholder +the MEPS-IC employer share of the premium for their coverage tier, employer +size and state (or census division for State and local government +employers). Those cells are published only as PDF tables, so this tool pins +the official AHRQ PDFs by URL, byte length and SHA-256, extracts the cells the +stage reads with ``pdftotext -layout`` (poppler), and writes them, with their +provenance, to +``packages/microcosm-build/src/microcosm/build/us_runtime/data/ +meps_ic_esi_premium_cells.json``. + +Every value is copied as published. Suppressed cells (``--``) stay ``null``: +the fallback for them is stage logic, declared in the stage manifest, not +data. Each table's United States row is checked against the AHRQ national +figure it must reproduce, and a parse that does not find exactly one row per +state, division or firm-size label refuses. + +Run (downloads the four PDFs, about 34 MB, into ``--pdf-dir``):: + + uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic + +``--check`` regenerates in memory and fails if the committed file differs. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import subprocess +import sys +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +OUTPUT_PATH = ( + REPOSITORY_ROOT + / "packages" + / "microcosm-build" + / "src" + / "microcosm" + / "build" + / "us_runtime" + / "data" + / "meps_ic_esi_premium_cells.json" +) +_MEPS_ROOT = "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/" + + +@dataclass(frozen=True) +class PinnedPdf: + key: str + url: str + size_bytes: int + sha256: str + survey_year: int + description: str + + @property + def filename(self) -> str: + return self.url.rsplit("/", 1)[-1] + + +PDFS: tuple[PinnedPdf, ...] = ( + PinnedPdf( + key="private_state_2025", + url=_MEPS_ROOT + "state/series_2/2025/ic25_iia_f.pdf", + size_bytes=12_530_005, + sha256="ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", + survey_year=2025, + description=( + "MEPS-IC 2025 Series II (private sector, by firm size and State): " + "Tables II.C/D/E.1 (average total premium per enrolled employee) " + "and II.C/D/E.2 (average employee contribution)" + ), + ), + PinnedPdf( + key="private_state_2024", + url=_MEPS_ROOT + "state/series_2/2024/ic24_iia_f.pdf", + size_bytes=10_687_163, + sha256="4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", + survey_year=2024, + description=( + "MEPS-IC 2024 Series II (private sector, by firm size and State); " + "the United States rows age the 2024 State and local government " + "cells to 2025" + ), + ), + PinnedPdf( + key="public_division_2024", + url=_MEPS_ROOT + "national/series_3/2024/ic24_iiia_g.pdf", + size_bytes=1_765_717, + sha256="462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79", + survey_year=2024, + description=( + "MEPS-IC 2024 Series III (State and local government, by " + "government type and size and census division): Tables " + "III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year " + "for this series (the 2025 series is not yet released)." + ), + ), + PinnedPdf( + key="private_national_2025", + url=_MEPS_ROOT + "national/series_1/2025/ic25_ia_g.pdf", + size_bytes=9_210_366, + sha256="cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3", + survey_year=2025, + description=( + "MEPS-IC 2025 Series I (private sector, national): Table I.A.2 " + "(percent of establishments that offer health insurance) and " + "Table I.A.2.j (percent of establishments that offer pretax " + "employee contributions to health insurance), by firm size" + ), + ), +) + +TIERS = ("single", "employee_plus_one", "family") +_TIER_LETTER = {"single": "C", "employee_plus_one": "E", "family": "D"} +_SERIES_II_COLUMNS = ( + "total", + "lt10", + "10_24", + "25_99", + "100_999", + "1000plus", + "lt50", + "50plus", +) +#: The columns the stage reads. The finer firm-size columns cannot be used: +#: CPS ASEC NOEMP bands (under 10, 10-49, 50-99, 100-499, 500-999, 1000+) do +#: not nest inside MEPS-IC's 10-24 / 25-99 / 100-999 bands. +_KEPT_SERIES_II_COLUMNS = ("total", "lt50", "50plus") +_SERIES_III_COLUMNS = ( + "all_state_and_local", + "state", + "local_lt250", + "local_250_999", + "local_1000_4999", + "local_5000_9999", + "local_10000plus", +) +_KEPT_SERIES_III_COLUMNS = ("all_state_and_local", "state") +DIVISIONS = ( + "New England", + "Middle Atlantic", + "East North Central", + "West North Central", + "South Atlantic", + "East South Central", + "West South Central", + "Mountain", + "Pacific", +) +STATE_FIPS = { + "Alabama": 1, + "Alaska": 2, + "Arizona": 4, + "Arkansas": 5, + "California": 6, + "Colorado": 8, + "Connecticut": 9, + "Delaware": 10, + "District of Columbia": 11, + "Florida": 12, + "Georgia": 13, + "Hawaii": 15, + "Idaho": 16, + "Illinois": 17, + "Indiana": 18, + "Iowa": 19, + "Kansas": 20, + "Kentucky": 21, + "Louisiana": 22, + "Maine": 23, + "Maryland": 24, + "Massachusetts": 25, + "Michigan": 26, + "Minnesota": 27, + "Mississippi": 28, + "Missouri": 29, + "Montana": 30, + "Nebraska": 31, + "Nevada": 32, + "New Hampshire": 33, + "New Jersey": 34, + "New Mexico": 35, + "New York": 36, + "North Carolina": 37, + "North Dakota": 38, + "Ohio": 39, + "Oklahoma": 40, + "Oregon": 41, + "Pennsylvania": 42, + "Rhode Island": 44, + "South Carolina": 45, + "South Dakota": 46, + "Tennessee": 47, + "Texas": 48, + "Utah": 49, + "Vermont": 50, + "Virginia": 51, + "Washington": 53, + "West Virginia": 54, + "Wisconsin": 55, + "Wyoming": 56, +} + +#: United States rows each extracted table must reproduce: the AHRQ national +#: figures (MEPS-IC Research Findings and the national dashboard). A parse +#: that drifts from them refuses. +EXPECTED_NATIONAL = { + ("private_state_2025", "II.C.1"): 9_025.0, + ("private_state_2025", "II.D.1"): 26_281.0, + ("private_state_2024", "II.C.1"): 8_486.0, + ("private_state_2024", "II.C.2"): 1_789.0, + ("private_state_2024", "II.E.1"): 16_931.0, + ("private_state_2024", "II.E.2"): 4_707.0, + ("private_state_2024", "II.D.1"): 24_540.0, + ("private_state_2024", "II.D.2"): 7_216.0, +} + +_NUMBER = re.compile(r"(--|\d[\d,]*(?:\.\d+)?%?)(\s*\*)?") + + +class ExtractionError(ValueError): + """The PDF text did not parse into the reviewed table shape.""" + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def fetch(pin: PinnedPdf, pdf_dir: Path) -> Path: + """Return the verified local copy of one pinned PDF, downloading it once.""" + + pdf_dir.mkdir(parents=True, exist_ok=True) + path = pdf_dir / pin.filename + if not path.exists(): + request = urllib.request.Request( + pin.url, headers={"User-Agent": "microcosm-build/meps-ic-esi-cells"} + ) + temporary = path.with_suffix(".pdf.part") + with urllib.request.urlopen(request, timeout=300) as response: # noqa: S310 + temporary.write_bytes(response.read()) + temporary.rename(path) + size = path.stat().st_size + digest = _sha256(path) + if size != pin.size_bytes or digest != pin.sha256: + raise ExtractionError( + f"{pin.filename}: expected {pin.size_bytes} bytes / {pin.sha256}, " + f"found {size} bytes / {digest}. AHRQ reissued the file; review " + "the new tables before re-pinning." + ) + return path + + +def pdf_text(path: Path) -> list[str]: + completed = subprocess.run( + ["pdftotext", "-layout", str(path), "-"], + check=True, + capture_output=True, + text=True, + ) + return [line.lstrip("\f") for line in completed.stdout.splitlines()] + + +def pdftotext_version() -> str: + completed = subprocess.run( + ["pdftotext", "-v"], check=True, capture_output=True, text=True + ) + return (completed.stderr or completed.stdout).splitlines()[0].strip() + + +def _table_block(lines: list[str], table_id: str) -> tuple[int, list[str]]: + head = re.compile(rf"^Table {re.escape(table_id)} (Average|Percent)") + starts = [ + index + for index, line in enumerate(lines) + if head.match(line) and "Standard errors" not in line + ] + if len(starts) != 1: + raise ExtractionError(f"Table {table_id}: found {len(starts)} headers.") + start = starts[0] + end = next( + ( + index + for index in range(start + 1, len(lines)) + if lines[index].startswith("Table ") + ), + len(lines), + ) + return start, lines[start:end] + + +def _row_values(block: list[str], label: str, width: int, table_id: str): + pattern = re.compile(rf"^\s*{re.escape(label)}\s{{2,}}") + rows = [line for line in block if pattern.match(line)] + if len(rows) != 1: + raise ExtractionError(f"Table {table_id}: {label!r} matched {len(rows)} rows.") + tokens = _NUMBER.findall(rows[0].split(label, 1)[1]) + if len(tokens) != width: + raise ExtractionError( + f"Table {table_id}: {label!r} has {len(tokens)} values, not {width}: " + f"{rows[0]!r}." + ) + values: list[float | None] = [] + unreliable: list[bool] = [] + for token, star in tokens: + values.append( + None if token == "--" else float(token.replace(",", "").replace("%", "")) + ) + unreliable.append(bool(star.strip())) + return values, unreliable + + +def _table_year(block: list[str], table_id: str) -> int: + match = re.search(r"United States, (\d{4})", " ".join(block[:3])) + if match is None: + raise ExtractionError(f"Table {table_id}: no survey year in the title.") + return int(match.group(1)) + + +def _state_divisions(block: list[str]) -> dict[str, str]: + """State -> census division from the table's own division headings.""" + + division = None + membership: dict[str, str] = {} + for line in block: + stripped = line.strip() + if stripped.endswith(":") and stripped[:-1] in DIVISIONS: + division = stripped[:-1] + continue + for state in STATE_FIPS: + if re.match(rf"^{re.escape(state)}\s{{2,}}", stripped): + if division is None: + raise ExtractionError(f"{state} precedes any division heading.") + membership[state] = division + if set(membership) != set(STATE_FIPS): + raise ExtractionError( + "Division headings did not place every state: missing " + f"{sorted(set(STATE_FIPS) - set(membership))}." + ) + return membership + + +def _series_ii_table(lines, pin: PinnedPdf, table_id: str) -> dict: + start, block = _table_block(lines, table_id) + if _table_year(block, table_id) != pin.survey_year: + raise ExtractionError(f"Table {table_id} is not survey year {pin.survey_year}.") + rows = {} + for label in ("United States", *STATE_FIPS): + # Labels anchor at the line start, so "Virginia" cannot match the + # "West Virginia" row nor "Kansas" the "Arkansas" row. + values, unreliable = _row_values( + block, label, len(_SERIES_II_COLUMNS), table_id + ) + by_column = dict(zip(_SERIES_II_COLUMNS, values, strict=True)) + flags = dict(zip(_SERIES_II_COLUMNS, unreliable, strict=True)) + key = "US" if label == "United States" else f"{STATE_FIPS[label]:02d}" + rows[key] = {column: by_column[column] for column in _KEPT_SERIES_II_COLUMNS} + starred = [column for column in _KEPT_SERIES_II_COLUMNS if flags[column]] + if starred: + rows[key]["unreliable"] = starred + expected = EXPECTED_NATIONAL.get((pin.key, table_id)) + if expected is not None and rows["US"]["total"] != expected: + raise ExtractionError( + f"Table {table_id} ({pin.survey_year}) United States total is " + f"{rows['US']['total']}, not the published {expected}." + ) + return {"text_line": start + 1, "title": block[0].strip(), "rows": rows} + + +def _series_iii_table(lines, pin: PinnedPdf, table_id: str) -> dict: + start, block = _table_block(lines, table_id) + if _table_year(block, table_id) != pin.survey_year: + raise ExtractionError(f"Table {table_id} is not survey year {pin.survey_year}.") + rows = {} + for label in ("United States", *DIVISIONS): + values, _unreliable = _row_values( + block, label, len(_SERIES_III_COLUMNS), table_id + ) + by_column = dict(zip(_SERIES_III_COLUMNS, values, strict=True)) + kept = {column: by_column[column] for column in _KEPT_SERIES_III_COLUMNS} + if any(value is None for value in kept.values()): + raise ExtractionError(f"Table {table_id}: {label} has a suppressed cell.") + rows["US" if label == "United States" else label] = kept + return {"text_line": start + 1, "title": block[0].strip(), "rows": rows} + + +def _pretax_offer_shares(lines, pin: PinnedPdf) -> dict: + """Firm-size pretax-contribution and health-insurance offer rates.""" + + _start, offer = _table_block(lines, "I.A.2") + offer_values, _ = _row_values(offer, "United States", 8, "I.A.2") + offer_by_column = dict(zip(_SERIES_II_COLUMNS, offer_values, strict=True)) + start_j, pretax = _table_block(lines, "I.A.2.j") + out = {} + for key, label in ( + ("total", "United States"), + ("lt50", "Less than 50 employees"), + ("50plus", "50+ employees"), + ): + values, unreliable = _row_values(pretax, label, 3, "I.A.2.j") + if unreliable[0]: + raise ExtractionError(f"Table I.A.2.j {label} is flagged unreliable.") + out[key] = { + "pretax_contribution_offer_percent": values[0], + "health_insurance_offer_percent": offer_by_column[key], + } + return { + "text_line": start_j + 1, + "title": pretax[0].strip(), + "offer_title": offer[0].strip(), + "rows": out, + } + + +def build_payload(pdf_dir: Path) -> dict: + texts = {pin.key: pdf_text(fetch(pin, pdf_dir)) for pin in PDFS} + private_2025 = { + tier: { + "premium": _series_ii_table( + texts["private_state_2025"], + PDFS[0], + f"II.{_TIER_LETTER[tier]}.1", + ), + "employee_contribution": _series_ii_table( + texts["private_state_2025"], + PDFS[0], + f"II.{_TIER_LETTER[tier]}.2", + ), + } + for tier in TIERS + } + private_2024_national = {} + for tier in TIERS: + private_2024_national[tier] = {} + for measure, suffix in (("premium", "1"), ("employee_contribution", "2")): + table = _series_ii_table( + texts["private_state_2024"], + PDFS[1], + f"II.{_TIER_LETTER[tier]}.{suffix}", + ) + private_2024_national[tier][measure] = { + "text_line": table["text_line"], + "total": table["rows"]["US"]["total"], + } + public_2024 = { + tier: { + "premium": _series_iii_table( + texts["public_division_2024"], + PDFS[2], + f"III.{_TIER_LETTER[tier]}.1", + ), + "employee_contribution": _series_iii_table( + texts["public_division_2024"], + PDFS[2], + f"III.{_TIER_LETTER[tier]}.2", + ), + } + for tier in TIERS + } + _start, division_block = _table_block(texts["private_state_2025"], "II.C.1") + divisions = _state_divisions(division_block) + return { + "schema_version": 1, + "issue": "PolicyEngine/microcosm#454", + "description": ( + "MEPS-IC average total premium and average employee contribution " + "per enrolled employee, by coverage tier, for the " + "meps_esi_premiums source stage: private-sector cells by State and " + "firm size (2025), State and local government cells by census " + "division (2024, the latest published year), the 2024 private " + "national rows that age the government cells, and the 2025 " + "firm-size pretax-contribution offer rates. Values are copied as " + "published; null marks a cell AHRQ suppressed ('--'). Dollar " + "values are annual; percent values are percents." + ), + "generator": "tools/build_us_meps_ic_esi_cells.py", + "extraction": { + "method": "pdftotext -layout, one row per label, refusing any " + "label that does not match exactly one line", + "pdftotext": pdftotext_version(), + }, + "sources": { + pin.key: { + "url": pin.url, + "size_bytes": pin.size_bytes, + "sha256": pin.sha256, + "survey_year": pin.survey_year, + "publisher": ( + "Agency for Healthcare Research and Quality, Center for " + "Financing, Access and Cost Trends, Medical Expenditure " + "Panel Survey-Insurance Component" + ), + "description": pin.description, + } + for pin in PDFS + }, + "tiers": list(TIERS), + "state_census_division": { + f"{STATE_FIPS[state]:02d}": divisions[state] for state in STATE_FIPS + }, + "private_state_2025": private_2025, + "private_national_2024": private_2024_national, + "public_division_2024": public_2024, + "pretax_contribution_2025": _pretax_offer_shares( + texts["private_national_2025"], PDFS[3] + ), + } + + +def render(payload: dict) -> str: + return json.dumps(payload, indent=1, sort_keys=True) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--pdf-dir", + type=Path, + required=True, + help="Directory caching the pinned AHRQ PDFs (downloaded when absent).", + ) + parser.add_argument( + "--check", + action="store_true", + help="Fail if the committed cell table differs from a regeneration.", + ) + args = parser.parse_args(argv) + rendered = render(build_payload(args.pdf_dir)) + if args.check: + if OUTPUT_PATH.read_text() != rendered: + print(f"{OUTPUT_PATH} is stale; regenerate it.", file=sys.stderr) + return 1 + print(f"{OUTPUT_PATH} is current.") + return 0 + OUTPUT_PATH.write_text(rendered) + print(f"Wrote {OUTPUT_PATH}.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a077d08e9fce946eed199674aa459c6f3ea8e8f7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 12:04:39 -0400 Subject: [PATCH 2/8] Produce ESI employer premiums and pre-tax employee premiums (#454) The declared meps_esi_premiums stage could not run: every pinned ASEC input omits the Census current employment-based coverage fields, so employer_sponsored_insurance_premiums was a recorded parity gap and pre_tax_health_insurance_premiums had no producer at all. Source fields. asec_census_person_columns restores NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and NOEMP from the pinned Census person members by exact PERIDNUM join, for every pooled vintage (none of the four pinned inputs carries them). Domains are the Census API codebooks for survey years 2023-2026, checked against all four pinned members. Stage (us_runtime/esi_premiums.py), run in the base build's pre-clone enrichment, both build paths: - Employer premium: an employed current policyholder whose job has an employer takes the MEPS-IC cell of their tier and employer (private: State x firm size, 2025; State government and local/federal: census division, 2024 aged to 2025). Employer paid all -> premium; some -> premium minus employee contribution; none -> 0. Dependents, non-employed policyholders and the unincorporated self-employed carry 0. One factor scales the weighted total to BEA NIPA 7.8 line 17 for the build year ($977.034B for 2024 in the 2026-09-30 vintage). - Pre-tax premium: an employed policyholder with wages, a reported premium and an employee share pays PHIP_VAL through payroll with the MEPS-IC firm-size pretax-offer probability. The blake2b draw is a new registered site of the seed protocol. The stage refuses missing, non-integer or out-of-codebook source values and the Census universe relations the members satisfy; it never proxies or defaults. Anchor. NHE Table 24 ($1,047.0B) counts COBRA and retiree enrollees; the same cells over every current policyholder reproduce it to 1.5% ($1,031.0B), 11.5% of that being non-employed policyholders. An employed-only column therefore scales to BEA's compensation series and holds NHE as an upper bound. Gates. us_esi_premiums_signal_gate (structure, clone agreement, common cell multiple, raw-share band) runs in the base build before and after the clone and in the release tool on base and export. us_esi_premiums_anchor_gate runs on the calibrated export for both the dense and sparse defaults: red when either column is absent or zero-mass, when the employer total leaves 5% of BEA or exceeds NHE. Its verdict ships as the esi_premiums gate evidence; --allow-esi-premium-gaps records without failing a diagnostic build. Registers. The parity known-gap entry is removed; both inputs are required release columns with two binding reform-coverage probes; the cms_nhe ESI target families become deferred reviewed exclusions and BEA's series is recorded source-absent; the source manifest, spec bundle, seed-site inventory and spec identity pins are regenerated or re-pinned at their observed values. Measured on the real pinned pools (experiments/us-esi-454), before calibration: 74.0M weighted policyholders with a positive employer premium, scale factor 1.070, mean $13,209, pre-tax premiums $181.1B on income years 2023-2025; all gates pass before and after the PUF clone. Not wired: the multispine pool, whose operators run on CPS rows carrying part of the stacked weight. Its releases fail the new gates until then. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 93 +- experiments/us-esi-454/README.md | 36 + .../receipts/stage_on_pool_2022_2024.json | 591 ++++++++ .../receipts/stage_on_pool_2023_2025.json | 592 ++++++++ .../us-esi-454/run_stage_on_pinned_pool.py | 235 ++++ .../src/microcosm/build/source_manifest.py | 2 + .../build/spec_engine/field_usage.py | 22 +- .../build/spec_engine/inventory_coverage.py | 33 +- .../spec_engine/schema/sources.schema.json | 121 ++ .../src/microcosm/build/spec_engine/seeds.py | 10 + .../build/us/ecps_parity_known_gaps.json | 41 - .../us/release_input_coverage_manifest.json | 49 +- .../src/microcosm/build/us/source_stages.json | 129 +- .../src/microcosm/build/us/spec/sources.yaml | 156 ++- .../src/microcosm/build/us/spec/spine.yaml | 4 + .../build/us/target_parity_manifest.json | 37 +- .../microcosm/build/us_runtime/__init__.py | 44 +- .../us_runtime/asec_census_person_columns.py | 56 + .../build/us_runtime/esi_premiums.py | 1112 +++++++++++++++ .../us_runtime/release_input_coverage.py | 9 + .../build/us_runtime/source_runtime.py | 10 + .../us/test_spec_engine_coverage_tool.py | 18 +- .../us/test_us_esi_premiums_engine.py | 114 ++ .../shared/test_release_target_parity.py | 2 +- .../shared/test_spec_engine_compiler_ir.py | 8 +- .../shared/test_spec_engine_field_usage.py | 16 +- .../test_spec_engine_identity_contracts.py | 4 +- .../shared/test_spec_engine_loader.py | 2 +- .../shared/test_spec_engine_plan_lock.py | 2 +- .../shared/test_spec_engine_seeds.py | 2 + .../us/test_us_asec_census_person_columns.py | 10 +- .../us/test_us_bundle_core_contracts.py | 2 +- .../engine_free/us/test_us_chronicle_feed.py | 2 +- .../engine_free/us/test_us_esi_premiums.py | 1230 +++++++++++++++++ .../us/test_us_fiscal_refresh_builder.py | 53 +- .../us/test_us_other_health_insurance.py | 88 +- .../tests/engine_free/us/test_us_plan.py | 22 + .../us/test_us_post_export_scoring.py | 8 +- .../us/test_us_puf_support_base_builder.py | 39 + .../engine_free/us/test_us_spec_bundle.py | 2 +- .../engine_free/us/test_us_spine_blindness.py | 5 + .../us/test_us_multispine_pool_tool.py | 2 +- .../us/test_us_post_export_scoring.py | 2 +- .../spec_engine_inventory_coverage.py | 6 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_fiscal_refresh_release.py | 106 ++ tools/build_us_puf_support_base.py | 47 + ...uild_us_release_input_coverage_manifest.py | 83 +- tools/build_us_target_parity_manifest.py | 85 +- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 2 +- .../identity_contracts.py | 7 +- 52 files changed, 5021 insertions(+), 334 deletions(-) create mode 100644 experiments/us-esi-454/README.md create mode 100644 experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json create mode 100644 experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json create mode 100644 experiments/us-esi-454/run_stage_on_pinned_pool.py create mode 100644 packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py create mode 100644 packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py create mode 100644 packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 0fabd95a86..62fa7c72eb 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "0931cbd3aae2abcd8c072b78eba40c1c006b1bf7fa23c6a6681f9a895e47070e", + "documentation_sha256": "d16558ea01d0eef4edd936f4f5166d25411d054d610cacbc3cd6037ee2737c00", "field_usage": { - "authored_normative_field_count": 32404, + "authored_normative_field_count": 32452, "claim_count": 49, "claims": [ { @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1716, - "pointer_sha256": "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + "pointer_count": 1761, + "pointer_sha256": "bdecb4db8c7c0a6b1dac1b39b4490007627f83bee305abcc1456304fdbfb5258", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -544,8 +544,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + "pointer_count": 280, + "pointer_sha256": "6cb87c6d4e3cca9907cc85d78a0ebd391281dc70391d30111d57a8bc84848bb6", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/seed_site_bindings", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 826, - "pointer_sha256": "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", + "pointer_count": 843, + "pointer_sha256": "c51302fee9ab90cd34097f3d6cc8a765f2a12cb236e68a5772b87d0756f45242", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -750,8 +750,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 277, - "pointer_sha256": "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + "pointer_count": 280, + "pointer_sha256": "1f88dfbe50cabf8e57c36f717b0cc952b5c423c1c2d62ac4219ec3c3872ead9f", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_site_bindings", @@ -772,21 +772,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42184, - "consumed_field_count": 42184, + "configuration_field_count": 42252, + "consumed_field_count": 42252, "generation0_effect_counts": { - "legacy_behavior": 38498, - "no_generation0_effect": 3686 + "legacy_behavior": 38543, + "no_generation0_effect": 3709 }, "mode_counts": { - "compiler_semantic": 27717, + "compiler_semantic": 27740, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14010 + "legacy_behavior": 14055 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7", - "resolved_binding_field_count": 9780, + "pointer_inventory_sha256": "8afbe29a5ad75630b11e561e242d1882c94efc10648d287e80c6ab0a7f44f1bc", + "resolved_binding_field_count": 9800, "unused_field_count": 0 }, "inventory_coverage": { @@ -815,9 +815,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, - "seed_owner_rows": 54, - "seed_sites": 53, + "seed_owner_bindings": 113, + "seed_owner_rows": 55, + "seed_sites": 54, "seed_streams": 14, "source_operators": 16, "source_stages": 38, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "b4f9cbbee81eff7f59304a114ed105910140f0799797c87f26ae9eb5090060f8" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20" + "sha256": "b4f9cbbee81eff7f59304a114ed105910140f0799797c87f26ae9eb5090060f8" }, "status": "covered" }, @@ -1533,7 +1533,7 @@ "expected": { "groups": 12, "partition": "disjoint and exhaustive", - "sites": 53 + "sites": 54 }, "failures": [], "observed": { @@ -1548,6 +1548,7 @@ "source_joint_count_calibration", "snap_take_up_assignment", "pregnancy_assignment", + "pre_tax_health_premium_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", "immigration_ead_workers_assignment", @@ -1616,7 +1617,7 @@ "sipp_tip_training_cap" ] }, - "sites": 53 + "sites": 54 }, "status": "covered" }, @@ -1633,12 +1634,12 @@ "compiler_ir.seed_stream_map.owners" ], "expected": { - "owner_rows": 54, + "owner_rows": 55, "relation": "site-order-preserving owner aggregation" }, "failures": [], "observed": { - "owner_rows": 54 + "owner_rows": 55 }, "status": "covered" }, @@ -1656,13 +1657,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "b4d6ec9c11ce323d83319a3d0289e0cf7216a7936d6c6f08cef0a463ad369641", - "protocol_sha256": "114217a2918329b0601d94deb8ed83f73919584b85ca8aabe45a40665afb6560" + "map_sha256": "bdb924d0b21e55e387a533d179dc6c0f7d239bee79bd9de640729af17c9b2301", + "protocol_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d" }, "failures": [], "observed": { - "map_sha256": "b4d6ec9c11ce323d83319a3d0289e0cf7216a7936d6c6f08cef0a463ad369641", - "protocol_sha256": "114217a2918329b0601d94deb8ed83f73919584b85ca8aabe45a40665afb6560" + "map_sha256": "bdb924d0b21e55e387a533d179dc6c0f7d239bee79bd9de640729af17c9b2301", + "protocol_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d" }, "status": "covered" }, @@ -1677,7 +1678,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "114217a2918329b0601d94deb8ed83f73919584b85ca8aabe45a40665afb6560", + "implementation_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1698,7 +1699,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "114217a2918329b0601d94deb8ed83f73919584b85ca8aabe45a40665afb6560", + "implementation_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1732,12 +1733,12 @@ ], "expected": { "relation": "all site fields preserved exactly", - "sites": 53 + "sites": 54 }, "failures": [], "observed": { - "sha256": "a0366cd518f42a240b589af3ef6580aac44ad7ed70969bcc669b365d315146ab", - "sites": 53 + "sha256": "a64c6425f6926eb072d260dddb60cacf5bd145c786a9b9bf68823ec45c077945", + "sites": 54 }, "status": "covered" }, @@ -1752,12 +1753,12 @@ "compiler_ir.seed_stream_map.sites.owners" ], "expected": { - "bindings": 112, - "coverage": "all 53 sites" + "bindings": 113, + "coverage": "all 54 sites" }, "failures": [], "observed": { - "bindings": 112 + "bindings": 113 }, "status": "covered" }, @@ -1775,12 +1776,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "b7f2650fd828f593bb53c8f6d37cbf1e0318111b8a81d4bc279954ed25e33fcd", "stages": 37 }, "failures": [], "observed": { - "sha256": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "sha256": "b7f2650fd828f593bb53c8f6d37cbf1e0318111b8a81d4bc279954ed25e33fcd", "stages": 38 }, "status": "covered" @@ -1887,7 +1888,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "69942294839a762738095894b87f6e72ea4faac03ebc020a4406c4f27bbb5c40" }, "failures": [], "observed": { @@ -1911,7 +1912,7 @@ "alpha", "zeta" ], - "sha256": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007" + "sha256": "69942294839a762738095894b87f6e72ea4faac03ebc020a4406c4f27bbb5c40" }, "status": "covered" }, @@ -1975,7 +1976,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "61a5484eab9fae1d6651b3ae15f1793d4934ea2f6ed2c08d05716b745c221014" + "sha256": "df681e94538c72b5ac39c4c3847fa86cf51eae8c38f545720344b3c94b58897a" }, "status": "covered" }, @@ -2599,7 +2600,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f76f921d4f74392b106b8c0729ce250494e0a55455eaba7b910b19cdf7825178" + "spec_sha256": "e9360b25997322886028b7a8a8964007201e2b36b7bbd341d0b483093e026950" } }, "report_schema_version": 3, @@ -2609,7 +2610,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f76f921d4f74392b106b8c0729ce250494e0a55455eaba7b910b19cdf7825178" + "spec_sha256": "e9360b25997322886028b7a8a8964007201e2b36b7bbd341d0b483093e026950" }, "status": "pass" } diff --git a/experiments/us-esi-454/README.md b/experiments/us-esi-454/README.md new file mode 100644 index 0000000000..8f88435f22 --- /dev/null +++ b/experiments/us-esi-454/README.md @@ -0,0 +1,36 @@ +# ESI premium stage on the pinned ASEC pools (#454) + +`run_stage_on_pinned_pool.py` pools real pinned ASEC inputs the way the base +build's source construction does (`build_pooled_asec_unit_frame`, with the +Census person-column restoration), runs the `meps_esi_premiums` stage, its +signal gate and the release anchor gate, clones the frame for PUF support and +grades the clone, and writes what it measured to `receipts/`. + +``` +uv run python tools/fetch_us_asec_sources.py --all-pinned # prints the pinned paths +uv run python experiments/us-esi-454/run_stage_on_pinned_pool.py \ + --asec-h5 2023= --asec-h5 2024= \ + --asec-h5 2025= \ + --census-person-dir ~/.cache/microcosm/cps/asec_education +``` + +The script refuses an input whose SHA-256 is not its pin +(`asec_sources.ASEC_SOURCE_ARTIFACTS`, `spm_role_source.ASEC_SPM_ROLE_SOURCES`). +About 80 seconds per pool. + +| Receipt | Pool | +|---|---| +| `receipts/stage_on_pool_2023_2025.json` | the default pool (`docs/us-asec-source-pins.md`) | +| `receipts/stage_on_pool_2022_2024.json` | the historical Build J/N/P pool | + +Each receipt records the input digests, the columns the restoration added per +vintage with their observed code counts, the stage summary (scale factor, raw +and scaled totals, totals by tier and employer sector, weighted policyholder +counts), totals by vintage, both gate verdicts before and after the clone, and +`nhe_concept_check`: the same MEPS-IC cells applied to every current +policyholder, which is how NHE Table 24's broader concept was measured. + +These are pre-calibration figures at pooled ASEC weights. They are evidence +about the stage, not about a release: +`packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py` pins +the figures the docs and registers quote to these files. diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json new file mode 100644 index 0000000000..0b013b3e1f --- /dev/null +++ b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json @@ -0,0 +1,591 @@ +{ + "anchor_gate": { + "details": { + "anchor": { + "label": "Private group health insurance", + "line": 17, + "published": "2026-09-30", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000.0, + "2024": 977034000000.0, + "2025": 1027929000000.0 + } + }, + "anchor_value": 977034000000.0, + "cross_check": { + "employee_contribution": { + "2023": 357800000000.0, + "2024": 382100000000.0 + }, + "employer_contribution": { + "2023": 975700000000.0, + "2024": 1047000000000.0 + }, + "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "Table 24 Employer-Sponsored Private Health Insurance" + }, + "cross_check_employer_ratio": 0.9331747851002865, + "cross_check_pre_tax_ratio": 0.45697198844356723, + "employer_premium_positive_persons": 74057205.19599111, + "employer_premium_total": 977034000000.0, + "pre_tax_premium_total": 174608996784.28705, + "relative_error": 0.0, + "relative_tolerance": 0.05, + "time_period": 2024 + }, + "failures": [], + "passed": true + }, + "by_vintage": { + "2022": { + "employer_premium_positive_persons": 24684189.26891587, + "employer_premium_total": 328107608719.83057, + "mean_per_positive_person": 13292.217343877182, + "pre_tax_premium_total": 55854772150.73417, + "weighted_persons": 108650789.68666667 + }, + "2023": { + "employer_premium_positive_persons": 24692526.63040858, + "employer_premium_total": 325317655884.286, + "mean_per_positive_person": 13174.741522143822, + "pre_tax_premium_total": 58750093810.022896, + "weighted_persons": 108650789.68666664 + }, + "2024": { + "employer_premium_positive_persons": 24680489.296666663, + "employer_premium_total": 323608735395.8834, + "mean_per_positive_person": 13111.92543656701, + "pre_tax_premium_total": 60004130823.53, + "weighted_persons": 108650789.68666667 + } + }, + "inputs": [ + { + "census_archive_sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + "census_person_member": "pppub23.csv", + "census_person_member_sha256": "19b56537e50e7663f954361ef2bb5ce9cef8d9d45f156fe1a69a99b654198ffe", + "h5": "census_cps_2022.h5", + "h5_sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e", + "income_year": 2022 + }, + { + "census_archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "census_person_member": "pppub24.csv", + "census_person_member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "h5": "census_cps_2023.h5", + "h5_sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", + "income_year": 2023 + }, + { + "census_archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "census_person_member": "pppub25.csv", + "census_person_member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "h5": "census_cps_2024.h5", + "h5_sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", + "income_year": 2024 + } + ], + "issue": "PolicyEngine/microcosm#454", + "nhe_concept_check": { + "nhe_table_24_employer_contribution_2024": 1047000000000.0, + "non_employed_share_of_all": 0.1162675199483739, + "raw_all_policyholders_over_nhe": 0.9902231741950678, + "raw_all_policyholders_total": 1036763663382.236, + "raw_employed_universe_total": 916221723468.1926, + "raw_non_employed_or_no_employer_total": 120541939914.04333 + }, + "person_rows": 432523, + "pool_census_person_columns": [ + { + "archive_sha256": "d2e000250782adfbdd7f29c82b66d866591a30f0d330496698ec19f9c784ce11", + "columns_added": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT", + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [], + "income_year": 2022, + "joined_person_rows": 146133, + "member": "pppub23.csv", + "member_sha256": "19b56537e50e7663f954361ef2bb5ce9cef8d9d45f156fe1a69a99b654198ffe", + "member_values": { + "A_ENRLW": { + "0": 74477, + "1": 11613, + "2": 60043 + }, + "A_EXPRRP": { + "1": 38440, + "10": 2799, + "11": 132, + "12": 1735, + "13": 4312, + "14": 919, + "2": 18399, + "3": 11887, + "4": 16284, + "5": 43805, + "7": 2792, + "8": 2765, + "9": 1864 + }, + "A_FTPT": { + "0": 134520, + "1": 10164, + "2": 1449 + }, + "NOEMP": { + "0": 72947, + "1": 13820, + "2": 10422, + "3": 4923, + "4": 8424, + "5": 3859, + "6": 31738 + }, + "NOW_CHAMPVA": { + "1": 260, + "2": 145873 + }, + "NOW_GRPFTYP": { + "0": 106295, + "1": 18329, + "2": 21509 + }, + "NOW_GRPFTYP2": { + "0": 106295, + "1": 10677, + "2": 7652, + "3": 21509 + }, + "NOW_HIPAID": { + "0": 106295, + "1": 8470, + "2": 27889, + "3": 3479 + }, + "NOW_IHSFLG": { + "1": 687, + "2": 145446 + }, + "NOW_MCAID": { + "1": 28184, + "2": 117949 + }, + "NOW_MIL": { + "1": 3882, + "2": 142251 + }, + "NOW_NONM": { + "1": 8666, + "2": 137467 + }, + "NOW_OTHMT": { + "1": 275, + "2": 145858 + }, + "NOW_OWNGRP": { + "0": 67360, + "1": 39838, + "2": 38935 + }, + "NOW_VACARE": { + "1": 1271, + "2": 144862 + }, + "PEMLR": { + "0": 29948, + "1": 66156, + "2": 2539, + "3": 408, + "4": 2064, + "5": 22286, + "6": 5332, + "7": 17400 + }, + "PTOTVAL": { + "negative_rows": 66, + "nonzero_rows": 102532 + } + } + }, + { + "archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "columns_added": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT", + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [], + "income_year": 2023, + "joined_person_rows": 144265, + "member": "pppub24.csv", + "member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "member_values": { + "A_ENRLW": { + "0": 72991, + "1": 11620, + "2": 59654 + }, + "A_EXPRRP": { + "1": 37875, + "10": 2763, + "11": 147, + "12": 1752, + "13": 4387, + "14": 968, + "2": 18376, + "3": 11652, + "4": 16147, + "5": 42756, + "7": 2661, + "8": 2854, + "9": 1927 + }, + "A_FTPT": { + "0": 132645, + "1": 10174, + "2": 1446 + }, + "NOEMP": { + "0": 70794, + "1": 13932, + "2": 10549, + "3": 4902, + "4": 8667, + "5": 3820, + "6": 31601 + }, + "NOW_CHAMPVA": { + "1": 335, + "2": 143930 + }, + "NOW_GRPFTYP": { + "0": 105104, + "1": 17779, + "2": 21382 + }, + "NOW_GRPFTYP2": { + "0": 105104, + "1": 10393, + "2": 7386, + "3": 21382 + }, + "NOW_HIPAID": { + "0": 105104, + "1": 7843, + "2": 28047, + "3": 3271 + }, + "NOW_IHSFLG": { + "1": 651, + "2": 143614 + }, + "NOW_MCAID": { + "1": 26367, + "2": 117898 + }, + "NOW_MIL": { + "1": 4318, + "2": 139947 + }, + "NOW_NONM": { + "1": 8435, + "2": 135830 + }, + "NOW_OTHMT": { + "1": 331, + "2": 143934 + }, + "NOW_OWNGRP": { + "0": 67555, + "1": 39161, + "2": 37549 + }, + "NOW_VACARE": { + "1": 1178, + "2": 143087 + }, + "PEMLR": { + "0": 28915, + "1": 66185, + "2": 2326, + "3": 410, + "4": 2225, + "5": 22025, + "6": 5335, + "7": 16844 + }, + "PTOTVAL": { + "negative_rows": 59, + "nonzero_rows": 102389 + } + } + }, + { + "archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "columns_added": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT" + ], + "income_year": 2024, + "joined_person_rows": 142125, + "member": "pppub25.csv", + "member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "member_values": { + "A_ENRLW": { + "0": 72079, + "1": 11445, + "2": 58601 + }, + "A_EXPRRP": { + "1": 37382, + "10": 2659, + "11": 134, + "12": 1734, + "13": 4257, + "14": 844, + "2": 18380, + "3": 11334, + "4": 15918, + "5": 42053, + "7": 2570, + "8": 2876, + "9": 1984 + }, + "A_FTPT": { + "0": 130680, + "1": 10014, + "2": 1431 + }, + "NOEMP": { + "0": 69665, + "1": 13465, + "2": 10406, + "3": 4884, + "4": 8547, + "5": 3763, + "6": 31395 + }, + "NOW_CHAMPVA": { + "1": 368, + "2": 141757 + }, + "NOW_GRPFTYP": { + "0": 103571, + "1": 17404, + "2": 21150 + }, + "NOW_GRPFTYP2": { + "0": 103571, + "1": 10176, + "2": 7228, + "3": 21150 + }, + "NOW_HIPAID": { + "0": 103571, + "1": 7748, + "2": 27378, + "3": 3428 + }, + "NOW_IHSFLG": { + "1": 685, + "2": 141440 + }, + "NOW_MCAID": { + "1": 24844, + "2": 117281 + }, + "NOW_MIL": { + "1": 4413, + "2": 137712 + }, + "NOW_NONM": { + "1": 8486, + "2": 133639 + }, + "NOW_OTHMT": { + "1": 312, + "2": 141813 + }, + "NOW_OWNGRP": { + "0": 66737, + "1": 38554, + "2": 36834 + }, + "NOW_VACARE": { + "1": 1270, + "2": 140855 + }, + "PEMLR": { + "0": 28155, + "1": 65222, + "2": 2214, + "3": 408, + "4": 2422, + "5": 22064, + "6": 5081, + "7": 16559 + }, + "PTOTVAL": { + "negative_rows": 75, + "nonzero_rows": 101163 + } + } + } + ], + "pooled_income_years": [ + 2022, + 2023, + 2024 + ], + "puf_support_clone": { + "anchor_gate": { + "employer_premium_total": 977034000000.0, + "failures": [], + "passed": true, + "pre_tax_premium_total": 174608996784.28705 + }, + "person_rows": 865046, + "signal_gate": { + "failures": [], + "passed": true + } + }, + "seconds": 47.4, + "seed": 0, + "signal_gate": { + "failures": [], + "passed": true + }, + "summary": { + "anchor": { + "label": "Private group health insurance", + "line": 17, + "published": "2026-09-30", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000.0, + "2024": 977034000000.0, + "2025": 1027929000000.0 + } + }, + "cells_sha256": "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8", + "clone_disagreement_source_persons": 0, + "cross_check": { + "employee_contribution": { + "2023": 357800000000.0, + "2024": 382100000000.0 + }, + "employer_contribution": { + "2023": 975700000000.0, + "2024": 1047000000000.0 + }, + "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "Table 24 Employer-Sponsored Private Health Insurance" + }, + "employer_premium_by_sector": { + "federal": 40180113357.65264, + "private": 756015257851.7373, + "state_and_local": 180838628790.61008 + }, + "employer_premium_by_tier": { + "employee_plus_one": 210094674350.32474, + "family": 415669227862.2854, + "single": 351270097787.38983 + }, + "employer_premium_mean_per_positive_person": 13192.963431637696, + "employer_premium_outside_universe_rows": 0, + "employer_premium_positive_persons": 74057205.19599111, + "employer_premium_positive_share": 0.2272025370134953, + "employer_premium_total": 977034000000.0, + "employer_premium_where_employer_pays_none_rows": 0, + "employer_premium_without_raw_share_rows": 0, + "negative_rows": 0, + "nonfinite_rows": 0, + "pre_tax_eligible_weight": 60688921.39918899, + "pre_tax_ineligible_rows": 0, + "pre_tax_not_reported_premium_rows": 0, + "pre_tax_premium_positive_persons": 52528029.87619302, + "pre_tax_premium_positive_share": 0.16115247153342174, + "pre_tax_premium_total": 174608996784.28705, + "pre_tax_selected_share_of_eligible": 0.8655291388469951, + "pre_tax_shares": { + "50plus": 0.9261603375527426, + "lt50": 0.4953271028037383, + "total": 0.7215447154471545 + }, + "raw_employer_share_band": [ + 6000.0, + 20000.0 + ], + "raw_employer_share_mean_per_positive_person": 12371.810697465933, + "raw_employer_share_total": 916221723468.1926, + "scale_factor": 1.0663728822119753, + "scale_factor_spread": 2.220446049250313e-16, + "weighted_employed_policyholders": 79790597.17620382, + "weighted_employed_policyholders_by_payment": { + "all": 15092018.672620341, + "none": 5733391.980212707, + "some": 58965186.52337077 + }, + "weighted_policyholders": 92947051.14638808 + }, + "target_year": 2024 +} diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json new file mode 100644 index 0000000000..96d8dafad2 --- /dev/null +++ b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json @@ -0,0 +1,592 @@ +{ + "anchor_gate": { + "details": { + "anchor": { + "label": "Private group health insurance", + "line": 17, + "published": "2026-09-30", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000.0, + "2024": 977034000000.0, + "2025": 1027929000000.0 + } + }, + "anchor_value": 977034000000.0, + "cross_check": { + "employee_contribution": { + "2023": 357800000000.0, + "2024": 382100000000.0 + }, + "employer_contribution": { + "2023": 975700000000.0, + "2024": 1047000000000.0 + }, + "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "Table 24 Employer-Sponsored Private Health Insurance" + }, + "cross_check_employer_ratio": 0.9331747851002865, + "cross_check_pre_tax_ratio": 0.47395266086209464, + "employer_premium_positive_persons": 73968814.73011559, + "employer_premium_total": 977034000000.0, + "pre_tax_premium_total": 181097311715.40637, + "relative_error": 0.0, + "relative_tolerance": 0.05, + "time_period": 2024 + }, + "failures": [], + "passed": true + }, + "by_vintage": { + "2023": { + "employer_premium_positive_persons": 24692526.63040858, + "employer_premium_total": 326526181481.6512, + "mean_per_positive_person": 13223.684492438199, + "pre_tax_premium_total": 58750093810.022896, + "weighted_persons": 108650789.68666664 + }, + "2024": { + "employer_premium_positive_persons": 24680489.296666663, + "employer_premium_total": 324810912508.5698, + "mean_per_positive_person": 13160.635050798553, + "pre_tax_premium_total": 60004130823.53, + "weighted_persons": 108650789.68666667 + }, + "2025": { + "employer_premium_positive_persons": 24595798.803040348, + "employer_premium_total": 325696906009.779, + "mean_per_positive_person": 13241.973095401916, + "pre_tax_premium_total": 62343087081.853455, + "weighted_persons": 108650789.68666668 + } + }, + "inputs": [ + { + "census_archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "census_person_member": "pppub24.csv", + "census_person_member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "h5": "census_cps_2023.h5", + "h5_sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", + "income_year": 2023 + }, + { + "census_archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "census_person_member": "pppub25.csv", + "census_person_member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "h5": "census_cps_2024.h5", + "h5_sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", + "income_year": 2024 + }, + { + "census_archive_sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + "census_person_member": "pppub26.csv", + "census_person_member_sha256": "f78920699bf671cc211cb720ff8c6274392524a22138aedaf9e3954f91ba4334", + "h5": "census_cps_2025.h5", + "h5_sha256": "4c5a32188b6acfbcfbeb9f5d719d3847873887b72ebdbb19bc402c16e0a64e58", + "income_year": 2025 + } + ], + "issue": "PolicyEngine/microcosm#454", + "nhe_concept_check": { + "nhe_table_24_employer_contribution_2024": 1047000000000.0, + "non_employed_share_of_all": 0.11462618085949701, + "raw_all_policyholders_over_nhe": 0.9847292814494486, + "raw_all_policyholders_total": 1031011557677.5726, + "raw_employed_universe_total": 912830640398.9915, + "raw_non_employed_or_no_employer_total": 118180917278.58118 + }, + "person_rows": 421119, + "pool_census_person_columns": [ + { + "archive_sha256": "cdb39cdac34bef99dd0940ab28e306f692404c2eea44d85dfd634214872a0a09", + "columns_added": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT", + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [], + "income_year": 2023, + "joined_person_rows": 144265, + "member": "pppub24.csv", + "member_sha256": "21a2b9e0e4b08534563578a45acad77868af4ae9a7d46f23776b707d4a559aa7", + "member_values": { + "A_ENRLW": { + "0": 72991, + "1": 11620, + "2": 59654 + }, + "A_EXPRRP": { + "1": 37875, + "10": 2763, + "11": 147, + "12": 1752, + "13": 4387, + "14": 968, + "2": 18376, + "3": 11652, + "4": 16147, + "5": 42756, + "7": 2661, + "8": 2854, + "9": 1927 + }, + "A_FTPT": { + "0": 132645, + "1": 10174, + "2": 1446 + }, + "NOEMP": { + "0": 70794, + "1": 13932, + "2": 10549, + "3": 4902, + "4": 8667, + "5": 3820, + "6": 31601 + }, + "NOW_CHAMPVA": { + "1": 335, + "2": 143930 + }, + "NOW_GRPFTYP": { + "0": 105104, + "1": 17779, + "2": 21382 + }, + "NOW_GRPFTYP2": { + "0": 105104, + "1": 10393, + "2": 7386, + "3": 21382 + }, + "NOW_HIPAID": { + "0": 105104, + "1": 7843, + "2": 28047, + "3": 3271 + }, + "NOW_IHSFLG": { + "1": 651, + "2": 143614 + }, + "NOW_MCAID": { + "1": 26367, + "2": 117898 + }, + "NOW_MIL": { + "1": 4318, + "2": 139947 + }, + "NOW_NONM": { + "1": 8435, + "2": 135830 + }, + "NOW_OTHMT": { + "1": 331, + "2": 143934 + }, + "NOW_OWNGRP": { + "0": 67555, + "1": 39161, + "2": 37549 + }, + "NOW_VACARE": { + "1": 1178, + "2": 143087 + }, + "PEMLR": { + "0": 28915, + "1": 66185, + "2": 2326, + "3": 410, + "4": 2225, + "5": 22025, + "6": 5335, + "7": 16844 + }, + "PTOTVAL": { + "negative_rows": 59, + "nonzero_rows": 102389 + } + } + }, + { + "archive_sha256": "318845a2b5e0034eb2973898de1738f4df0025727de38499e7669cb9c0deef0b", + "columns_added": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT" + ], + "income_year": 2024, + "joined_person_rows": 142125, + "member": "pppub25.csv", + "member_sha256": "06921fe83fc66c907e6c7b86b82255dc70458ee7d76258fc48297cb34f0c06b5", + "member_values": { + "A_ENRLW": { + "0": 72079, + "1": 11445, + "2": 58601 + }, + "A_EXPRRP": { + "1": 37382, + "10": 2659, + "11": 134, + "12": 1734, + "13": 4257, + "14": 844, + "2": 18380, + "3": 11334, + "4": 15918, + "5": 42053, + "7": 2570, + "8": 2876, + "9": 1984 + }, + "A_FTPT": { + "0": 130680, + "1": 10014, + "2": 1431 + }, + "NOEMP": { + "0": 69665, + "1": 13465, + "2": 10406, + "3": 4884, + "4": 8547, + "5": 3763, + "6": 31395 + }, + "NOW_CHAMPVA": { + "1": 368, + "2": 141757 + }, + "NOW_GRPFTYP": { + "0": 103571, + "1": 17404, + "2": 21150 + }, + "NOW_GRPFTYP2": { + "0": 103571, + "1": 10176, + "2": 7228, + "3": 21150 + }, + "NOW_HIPAID": { + "0": 103571, + "1": 7748, + "2": 27378, + "3": 3428 + }, + "NOW_IHSFLG": { + "1": 685, + "2": 141440 + }, + "NOW_MCAID": { + "1": 24844, + "2": 117281 + }, + "NOW_MIL": { + "1": 4413, + "2": 137712 + }, + "NOW_NONM": { + "1": 8486, + "2": 133639 + }, + "NOW_OTHMT": { + "1": 312, + "2": 141813 + }, + "NOW_OWNGRP": { + "0": 66737, + "1": 38554, + "2": 36834 + }, + "NOW_VACARE": { + "1": 1270, + "2": 140855 + }, + "PEMLR": { + "0": 28155, + "1": 65222, + "2": 2214, + "3": 408, + "4": 2422, + "5": 22064, + "6": 5081, + "7": 16559 + }, + "PTOTVAL": { + "negative_rows": 75, + "nonzero_rows": 101163 + } + } + }, + { + "archive_sha256": "fe819d0d2fc4470c282e76c247b8aa8b6054811619730307e9f2fe6186707d93", + "columns_added": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP" + ], + "columns_verified_equal": [ + "NOW_MCAID", + "NOW_NONM", + "NOW_CHAMPVA", + "NOW_MIL", + "NOW_VACARE", + "NOW_OTHMT", + "NOW_IHSFLG", + "A_EXPRRP", + "PTOTVAL", + "A_ENRLW", + "A_FTPT" + ], + "income_year": 2025, + "joined_person_rows": 134729, + "member": "pppub26.csv", + "member_sha256": "f78920699bf671cc211cb720ff8c6274392524a22138aedaf9e3954f91ba4334", + "member_values": { + "A_ENRLW": { + "0": 69572, + "1": 10761, + "2": 54396 + }, + "A_EXPRRP": { + "1": 35882, + "10": 2404, + "11": 97, + "12": 1408, + "13": 3816, + "14": 800, + "2": 17462, + "3": 10808, + "4": 15562, + "5": 39531, + "7": 2452, + "8": 2717, + "9": 1790 + }, + "A_FTPT": { + "0": 123968, + "1": 9470, + "2": 1291 + }, + "NOEMP": { + "0": 66679, + "1": 12893, + "2": 9521, + "3": 4664, + "4": 8218, + "5": 3402, + "6": 29352 + }, + "NOW_CHAMPVA": { + "1": 414, + "2": 134315 + }, + "NOW_GRPFTYP": { + "0": 98544, + "1": 16429, + "2": 19756 + }, + "NOW_GRPFTYP2": { + "0": 98544, + "1": 9623, + "2": 6806, + "3": 19756 + }, + "NOW_HIPAID": { + "0": 98544, + "1": 7449, + "2": 25717, + "3": 3019 + }, + "NOW_IHSFLG": { + "1": 661, + "2": 134068 + }, + "NOW_MCAID": { + "1": 22688, + "2": 112041 + }, + "NOW_MIL": { + "1": 3994, + "2": 130735 + }, + "NOW_NONM": { + "1": 8102, + "2": 126627 + }, + "NOW_OTHMT": { + "1": 235, + "2": 134494 + }, + "NOW_OWNGRP": { + "0": 63878, + "1": 36185, + "2": 34666 + }, + "NOW_VACARE": { + "1": 1377, + "2": 133352 + }, + "PEMLR": { + "0": 25970, + "1": 61505, + "2": 1924, + "3": 382, + "4": 2253, + "5": 22471, + "6": 4889, + "7": 15335 + }, + "PTOTVAL": { + "negative_rows": 64, + "nonzero_rows": 96720 + } + } + } + ], + "pooled_income_years": [ + 2023, + 2024, + 2025 + ], + "puf_support_clone": { + "anchor_gate": { + "employer_premium_total": 977034000000.0, + "failures": [], + "passed": true, + "pre_tax_premium_total": 181097311715.4063 + }, + "person_rows": 842238, + "signal_gate": { + "failures": [], + "passed": true + } + }, + "seconds": 54.4, + "seed": 0, + "signal_gate": { + "failures": [], + "passed": true + }, + "summary": { + "anchor": { + "label": "Private group health insurance", + "line": 17, + "published": "2026-09-30", + "series": "B4923C", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "values": { + "2023": 923195000000.0, + "2024": 977034000000.0, + "2025": 1027929000000.0 + } + }, + "cells_sha256": "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8", + "clone_disagreement_source_persons": 0, + "cross_check": { + "employee_contribution": { + "2023": 357800000000.0, + "2024": 382100000000.0 + }, + "employer_contribution": { + "2023": 975700000000.0, + "2024": 1047000000000.0 + }, + "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "Table 24 Employer-Sponsored Private Health Insurance" + }, + "employer_premium_by_sector": { + "federal": 39589380243.32626, + "private": 756704453166.5498, + "state_and_local": 180740166590.12393 + }, + "employer_premium_by_tier": { + "employee_plus_one": 208020070710.58252, + "family": 414697085204.7131, + "single": 354316844084.70447 + }, + "employer_premium_mean_per_positive_person": 13208.728618470228, + "employer_premium_outside_universe_rows": 0, + "employer_premium_positive_persons": 73968814.73011559, + "employer_premium_positive_share": 0.2269313609943412, + "employer_premium_total": 977034000000.0, + "employer_premium_where_employer_pays_none_rows": 0, + "employer_premium_without_raw_share_rows": 0, + "negative_rows": 0, + "nonfinite_rows": 0, + "pre_tax_eligible_weight": 60693311.22982113, + "pre_tax_ineligible_rows": 0, + "pre_tax_not_reported_premium_rows": 0, + "pre_tax_premium_positive_persons": 52487359.3024551, + "pre_tax_premium_positive_share": 0.161027696941799, + "pre_tax_premium_total": 181097311715.40637, + "pre_tax_selected_share_of_eligible": 0.8647964370193382, + "pre_tax_shares": { + "50plus": 0.9261603375527426, + "lt50": 0.4953271028037383, + "total": 0.7215447154471545 + }, + "raw_employer_share_band": [ + 6000.0, + 20000.0 + ], + "raw_employer_share_mean_per_positive_person": 12340.74986505553, + "raw_employer_share_total": 912830640398.9915, + "scale_factor": 1.0703343607889255, + "scale_factor_spread": 4.440892098500626e-16, + "weighted_employed_policyholders": 79689927.57027404, + "weighted_employed_policyholders_by_payment": { + "all": 14922582.431638531, + "none": 5721112.840158437, + "some": 59046232.29847707 + }, + "weighted_policyholders": 92422322.37394994 + }, + "target_year": 2024 +} diff --git a/experiments/us-esi-454/run_stage_on_pinned_pool.py b/experiments/us-esi-454/run_stage_on_pinned_pool.py new file mode 100644 index 0000000000..64cec8a6a1 --- /dev/null +++ b/experiments/us-esi-454/run_stage_on_pinned_pool.py @@ -0,0 +1,235 @@ +"""Run the meps_esi_premiums stage on a real pinned ASEC pool. + +Pools sha-pinned processed ASEC H5 inputs exactly as the base build does, +restoring the reviewed Census person columns (#720, #454) from the pinned +official person members, then runs the stage, its signal gate, the release +anchor gate (pre-calibration) and the PUF-support clone, and writes the +measured lineage to ``receipts/stage_on_pool__.json``. + + uv run python experiments/us-esi-454/run_stage_on_pinned_pool.py \ + --asec-h5 2023= --asec-h5 2024=<...> \ + --asec-h5 2025=<...> \ + --census-person-dir ~/.cache/microcosm/cps/asec_education + +The default pool is income years 2023-2025 (``docs/us-asec-source-pins.md``); +2022-2024 is the historical Build J/N/P pool. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import hashlib +import json +import time +from pathlib import Path + +import numpy as np + +from microcosm.build.us_runtime import esi_premiums as esi +from microcosm.build.us_runtime.asec_pool import ( + AsecSource, + build_pooled_asec_unit_frame, +) +from microcosm.build.us_runtime.asec_sources import ASEC_SOURCE_ARTIFACTS +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PRE_TAX_PREMIUM_COLUMN, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + us_esi_premiums_summary, + with_us_esi_premium_inputs, +) +from microcosm.build.us_runtime.puf_support import clone_us_frame_for_puf_support +from microcosm.build.us_runtime.spm_role_source import ASEC_SPM_ROLE_SOURCES + +HERE = Path(__file__).resolve().parent + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def _jsonable(value): + if isinstance(value, dict): + return {str(key): _jsonable(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(item) for item in value] + if isinstance(value, (np.floating, np.integer)): + return value.item() + return value + + +def _by_vintage(frame, years) -> dict[str, dict[str, float]]: + person = frame.table("person") + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + out = {} + for year in years: + mask = person["source_year"].to_numpy() == year + employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy() + pre_tax = person[US_ESI_PRE_TAX_PREMIUM_COLUMN].to_numpy() + holders = mask & (employer > 0) + out[str(year)] = { + "weighted_persons": float(weights[mask].sum()), + "employer_premium_total": float(weights[mask] @ employer[mask]), + "employer_premium_positive_persons": float(weights[holders].sum()), + "mean_per_positive_person": float( + weights[holders] @ employer[holders] / weights[holders].sum() + ), + "pre_tax_premium_total": float(weights[mask] @ pre_tax[mask]), + } + return out + + +def _all_policyholder_concept_check(frame) -> dict[str, float]: + """Raw MEPS-IC shares over EVERY current policyholder (NHE's concept). + + Non-employed policyholders (retirees, COBRA, others) take their State's + all-sizes private cell; this is diagnostics only, never the column. + """ + + person = esi._person_with_state(frame) + codes = esi._esi_person_codes(person) + holder = codes.owner == 1 + widened = dataclasses.replace( + codes, + employed=np.ones(len(codes.owner), dtype=bool), + sector=np.where(np.isin(codes.sector, (1, 2, 3, 4, 5, 6)), codes.sector, 4), + noemp=np.where(codes.employed, codes.noemp, 0), + ) + premium, contribution = esi._cell_values( + widened, esi.load_meps_ic_esi_premium_cells() + ) + raw = esi._raw_employer_share(widened, premium, contribution) + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + employed = codes.universe + total = float(weights[holder] @ raw[holder]) + employed_total = float(weights[employed] @ raw[employed]) + nhe = float(esi.EMPLOYER_PREMIUM_CROSS_CHECK["employer_contribution"]["2024"]) + return { + "raw_all_policyholders_total": total, + "raw_employed_universe_total": employed_total, + "raw_non_employed_or_no_employer_total": total - employed_total, + "non_employed_share_of_all": (total - employed_total) / total, + "nhe_table_24_employer_contribution_2024": nhe, + "raw_all_policyholders_over_nhe": total / nhe, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--asec-h5", + action="append", + required=True, + help="Pinned processed ASEC input as YEAR=PATH; once per pooled year.", + ) + parser.add_argument("--census-person-dir", type=Path, required=True) + parser.add_argument("--target-year", type=int, default=2024) + parser.add_argument("--seed", type=int, default=0) + args = parser.parse_args() + paths = { + int(year): Path(path).expanduser() + for year, path in (value.split("=", 1) for value in args.asec_h5) + } + years = tuple(sorted(paths)) + started = time.perf_counter() + sources = [] + inputs = [] + for year in years: + artifact = ASEC_SOURCE_ARTIFACTS[year] + h5 = paths[year] + digest = _sha256(h5) + if digest != artifact.sha256: + raise SystemExit( + f"{h5}: sha256 {digest} is not the pinned {artifact.sha256}" + ) + pin = ASEC_SPM_ROLE_SOURCES[year] + member = args.census_person_dir / pin.member + member_digest = _sha256(member) + if member_digest != pin.csv_sha256: + raise SystemExit( + f"{member}: sha256 {member_digest} is not {pin.csv_sha256}" + ) + sources.append(AsecSource(year=year, path=h5, census_person_source=member)) + inputs.append( + { + "income_year": year, + "h5": artifact.filename, + "h5_sha256": digest, + "census_person_member": pin.member, + "census_person_member_sha256": member_digest, + "census_archive_sha256": pin.archive_sha256, + } + ) + frame, metadata = build_pooled_asec_unit_frame( + sources, target_year=args.target_year + ) + staged = with_us_esi_premium_inputs( + frame, seed=args.seed, time_period=args.target_year + ) + signal = us_esi_premiums_signal_gate(staged) + anchor = us_esi_premiums_anchor_gate(staged, time_period=args.target_year) + cloned = clone_us_frame_for_puf_support(staged) + cloned_signal = us_esi_premiums_signal_gate(cloned) + cloned_anchor = us_esi_premiums_anchor_gate(cloned, time_period=args.target_year) + receipt = { + "issue": "PolicyEngine/microcosm#454", + "inputs": inputs, + "seed": args.seed, + "target_year": args.target_year, + "pooled_income_years": list(years), + "pool_census_person_columns": [ + { + key: _jsonable(source["census_person_columns"][key]) + for key in ( + "income_year", + "member", + "member_sha256", + "archive_sha256", + "joined_person_rows", + "columns_added", + "columns_verified_equal", + "member_values", + ) + } + for source in metadata["sources"] + ], + "summary": _jsonable(us_esi_premiums_summary(staged)), + "by_vintage": _by_vintage(staged, years), + "nhe_concept_check": _all_policyholder_concept_check(staged), + "signal_gate": {"passed": signal.passed, "failures": list(signal.failures)}, + "anchor_gate": { + "passed": anchor.passed, + "failures": list(anchor.failures), + "details": _jsonable(dict(anchor.details)), + }, + "puf_support_clone": { + "signal_gate": { + "passed": cloned_signal.passed, + "failures": list(cloned_signal.failures), + }, + "anchor_gate": { + "passed": cloned_anchor.passed, + "failures": list(cloned_anchor.failures), + "employer_premium_total": cloned_anchor.details.get( + "employer_premium_total" + ), + "pre_tax_premium_total": cloned_anchor.details.get( + "pre_tax_premium_total" + ), + }, + "person_rows": len(cloned.table("person")), + }, + "person_rows": len(staged.table("person")), + "seconds": round(time.perf_counter() - started, 1), + } + out = HERE / "receipts" / f"stage_on_pool_{years[0]}_{years[-1]}.json" + out.write_text(json.dumps(receipt, indent=1, sort_keys=True) + "\n") + print(json.dumps({k: receipt[k] for k in ("signal_gate", "anchor_gate")}, indent=1)) + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 2588173e1b..86850f5fc5 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -87,11 +87,13 @@ "derive_energy_subsidy", "derive_education_inputs", "derive_eligibility_inputs", + "derive_employer_sponsored_insurance_premiums", "derive_hours_worked", "derive_housing_tenure_inputs", "derive_immigration_status", "derive_medicare_take_up", "derive_other_health_insurance_premiums", + "derive_pre_tax_health_insurance_premiums", "derive_prior_year_income", "derive_snap_abawd_discretionary_exemption", "derive_snap_take_up", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 56f39195cf..7086d370ab 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_404 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_184 +EXPECTED_AUTHORED_FIELD_COUNT = 32_452 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_800 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_252 class FieldUsageError(AssertionError): @@ -453,12 +453,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "6a781915fd491d2c4b16d2b7d482f69cf362c904130093c59f9629f7a319269b", ), "resolved_seed_protocol": ( - 826, - "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", + 843, + "c51302fee9ab90cd34097f3d6cc8a765f2a12cb236e68a5772b87d0756f45242", ), "resolved_seed_site_bindings": ( - 277, - "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", + 280, + "1f88dfbe50cabf8e57c36f717b0cc952b5c423c1c2d62ac4219ec3c3872ead9f", ), "resolved_vintage_authorities": ( 63, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_716, - "3e808ed9f5a791c0dfb85be0c59adf803366bf6e8c7a9f5a58f92974e044b607", + 1_761, + "bdecb4db8c7c0a6b1dac1b39b4490007627f83bee305abcc1456304fdbfb5258", ), "spine_assembly_household_mass_shares": ( 2, @@ -513,8 +513,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "29c6c1b3243e178783e7ab139993ba3a9b42d62edd4e9e4e1f3b28688daf2c6d", ), "spine_seed_site_bindings": ( - 277, - "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", + 280, + "6cb87c6d4e3cca9907cc85d78a0ebd391281dc70391d30111d57a8bc84848bb6", ), "spine_support_roles": ( 29, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 673ab08d35..0ac360e9e9 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -294,6 +294,7 @@ "source_joint_count_calibration", "snap_take_up_assignment", "pregnancy_assignment", + "pre_tax_health_premium_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", "immigration_ead_workers_assignment", @@ -350,18 +351,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "3db095ce2d1ef89028d3862f0edc827d704f099c5b3323f9b115642a3c27e007", + "full_checkpoint": "69942294839a762738095894b87f6e72ea4faac03ebc020a4406c4f27bbb5c40", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "cc34ba11288d01145fc2dc1a669df3f7c4ef2019d73afb02dd8eeee852b16d20", + "late_resource_semantics": "b4f9cbbee81eff7f59304a114ed105910140f0799797c87f26ae9eb5090060f8", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "seed_map": "b4d6ec9c11ce323d83319a3d0289e0cf7216a7936d6c6f08cef0a463ad369641", - "seed_protocol": "114217a2918329b0601d94deb8ed83f73919584b85ca8aabe45a40665afb6560", - "source_manifest": "f4f8bfeb79268a4043a843b47cec1d1c0e58e9f5dd6bc23417ada95446dd7b8e", + "seed_map": "bdb924d0b21e55e387a533d179dc6c0f7d239bee79bd9de640729af17c9b2301", + "seed_protocol": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d", + "source_manifest": "b7f2650fd828f593bb53c8f6d37cbf1e0318111b8a81d4bc279954ed25e33fcd", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -434,9 +435,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, - "seed_owner_rows": 54, - "seed_sites": 53, + "seed_owner_bindings": 113, + "seed_owner_rows": 55, + "seed_sites": 54, "seed_streams": 14, "source_operators": 16, "source_stages": 38, @@ -1778,7 +1779,7 @@ def add( "seed_site_definitions_exact", clauses={ "seed site definitions differ": site_definitions_exact, - "seed site count differs": len(protocol_sites) == 53, + "seed site count differs": len(protocol_sites) == 54, }, homes=("/bundle/seed_protocol",), consumers=("compiler_ir.seed_stream_map.sites", "compiler_ir.node_slices"), @@ -1786,7 +1787,7 @@ def add( "sites": len(protocol_sites), "sha256": sha256_json([site.to_wire() for site in protocol.sites]), }, - expected={"sites": 53, "relation": "all site fields preserved exactly"}, + expected={"sites": 54, "relation": "all site fields preserved exactly"}, ) binding_by_site = {binding.site: binding for binding in spec.seed_site_bindings} binding_exact = set(binding_by_site) == set(compiled_sites) and all( @@ -1801,7 +1802,7 @@ def add( "owner binding count differs": sum( len(site.owners) for site in compiled_sites.values() ) - == 112, + == 113, "one or more sites have no owner": all( site.owners for site in compiled_sites.values() ), @@ -1811,7 +1812,7 @@ def add( observed={ "bindings": sum(len(site.owners) for site in compiled_sites.values()) }, - expected={"bindings": 112, "coverage": "all 53 sites"}, + expected={"bindings": 113, "coverage": "all 54 sites"}, ) expected_owner_sites: dict[tuple[str, str], list[str]] = {} for site in protocol.sites: @@ -1835,13 +1836,13 @@ def add( "seed owner rows differ": _json_equal( compiled_owner_rows, expected_owner_rows ), - "seed owner row count differs": len(compiled_owner_rows) == 54, + "seed owner row count differs": len(compiled_owner_rows) == 55, }, homes=("/spine/seed_site_bindings", "/bundle/seed_protocol"), consumers=("compiler_ir.seed_stream_map.owners",), observed={"owner_rows": len(compiled_owner_rows)}, expected={ - "owner_rows": 54, + "owner_rows": 55, "relation": "site-order-preserving owner aggregation", }, ) @@ -1855,7 +1856,7 @@ def add( "seed groups overlap": groups_disjoint, "seed groups do not cover the protocol exactly": set(grouped_ids) == set(protocol_sites), - "seed group cardinality differs": len(grouped_ids) == 53, + "seed group cardinality differs": len(grouped_ids) == 54, }, homes=("/bundle/seed_protocol", "/spine/seed_site_bindings"), consumers=("compiler_ir.seed_stream_map",), @@ -1865,7 +1866,7 @@ def add( }, expected={ "groups": len(EXPECTED_SEED_GROUPS), - "sites": 53, + "sites": 54, "partition": "disjoint and exhaustive", }, ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 839249b199..6521834923 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1714,6 +1714,127 @@ "seed_from_build_config" ] }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "anchor": { + "type": "object", + "additionalProperties": false, + "properties": { + "series": { + "type": "string" + }, + "sha256": { + "$ref": "defs.schema.json#/$defs/sha256" + }, + "source": { + "type": "string", + "x-spec-surface": "operational" + }, + "values": { + "type": "object", + "additionalProperties": { + "type": "number" + } + } + }, + "required": [ + "series", + "sha256", + "source", + "values" + ] + }, + "cells_resource": { + "type": "string" + }, + "cells_sha256": { + "$ref": "defs.schema.json#/$defs/sha256" + }, + "government_cell_aging": { + "type": "string", + "x-spec-surface": "documentation" + }, + "kind": { + "const": "derive_employer_sponsored_insurance_premiums" + }, + "payment_column": { + "type": "string" + }, + "payment_rule": { + "type": "string", + "x-spec-surface": "documentation" + }, + "sector_column": { + "type": "string" + }, + "size_column": { + "type": "string" + }, + "suppressed_cell_fallback": { + "type": "string", + "x-spec-surface": "documentation" + }, + "tier_column": { + "type": "string" + }, + "universe": { + "type": "string", + "x-spec-surface": "documentation" + } + }, + "required": [ + "anchor", + "cells_resource", + "cells_sha256", + "government_cell_aging", + "kind", + "payment_column", + "payment_rule", + "sector_column", + "size_column", + "suppressed_cell_fallback", + "tier_column", + "universe" + ] + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "employer_payment_codes": { + "type": "array", + "items": { + "type": "integer" + } + }, + "kind": { + "const": "derive_pre_tax_health_insurance_premiums" + }, + "pre_tax_share": { + "type": "string", + "x-spec-surface": "documentation" + }, + "premium_column": { + "type": "string" + }, + "seed_from_build_config": { + "type": "boolean" + }, + "wage_column": { + "type": "string" + } + }, + "required": [ + "employer_payment_codes", + "kind", + "pre_tax_share", + "premium_column", + "seed_from_build_config", + "wage_column" + ] + }, { "type": "object", "additionalProperties": false, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py index e837a009e5..1151c3018f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py @@ -315,6 +315,7 @@ def _site( "microcosm.build.us_runtime.congressional_district_geography", "microcosm.build.us_runtime.disability_benefits", "microcosm.build.us_runtime.energy_subsidy", + "microcosm.build.us_runtime.esi_premiums", "microcosm.build.us_runtime.geography_ladder", "microcosm.build.us_runtime.housing_inputs", "microcosm.build.us_runtime.immigration", @@ -724,6 +725,15 @@ def _stable_site( ), candidate_universe="all_person_rows_before_sex_and_age_eligibility_mask", ), + _stable_site( + "pre_tax_health_premium_assignment", + salt="pre_tax_health_insurance_premiums", + key_grammar=( + "source_year:source_household_id:source_person_id_if_complete", + "else_person_id", + ), + candidate_universe="all_person_rows_before_pre_tax_eligibility_mask", + ), _stable_site( "wic_claim_assignment", salt="would_claim_wic", diff --git a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json index 97c3bda958..cd769fa333 100644 --- a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json @@ -2,47 +2,6 @@ "schema_version": 1, "description": "Reason'd, issue-linked parity exemption register (the debt ledger). Each entry names a layer the incumbent eCPS populates that the current Microcosm candidate does not, with the reason it is absent and the tracking issue that owns closing it. parity_gate is run with known_gaps = the keys of this file, and fails on any entry whose layer the candidate now populates (stale exemptions cannot rot silently); the launch condition is that this register shrinks to deliberate scope decisions, not TODOs. Seeded from the first parity run of populace_us_2024.h5 against the pinned reference (see ecps_parity_reference.json). Every issue ref is an existing open PolicyEngine/microcosm issue.", "known_gaps": { - "employer_sponsored_insurance_premiums": { - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 197-271 and 1575-1581 requires NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL; datasets/cps/census_cps.py lines 13-55 declares the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs consumed by the hermetic --asec-h5 build contain PHIP_VAL but omit all three required NOW_* fields, so the retired owner/payment/plan-type derivation cannot execute. has_esi and NOW_GRP are not semantically valid substitutes.", - "issue": "PolicyEngine/microcosm#32", - "evidence": { - "classification": "source_unavailability", - "retired_derivation": { - "repository_owner": "PolicyEngine", - "repository_name_parts": ["policyengine-", "us-data"], - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "cps", "cps.py"], - "lines": "197-271,1575-1581" - }, - "optional_source_fields": { - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "cps", "census_cps.py"], - "lines": "13-55" - }, - "required_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "PHIP_VAL"], - "hermetic_inputs": [ - { - "filename": "census_cps_2022.h5", - "sha256": "7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e", - "present_columns": ["PHIP_VAL"], - "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] - }, - { - "filename": "census_cps_2023.h5", - "sha256": "cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88", - "present_columns": ["PHIP_VAL"], - "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] - }, - { - "filename": "census_cps_2024.h5", - "sha256": "ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d", - "present_columns": ["PHIP_VAL"], - "missing_columns": ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP"] - } - ], - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88, 647-687, and 748-755 consumes those mappings and does not refresh omitted raw Census columns." - } - }, "financial_assistance": { "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1493-1496 derives person-level financial_assistance directly from FIN_VAL; datasets/cps/census_cps.py lines 39-58 and 306-359 make FIN_VAL a required person source. All three SHA-recorded 2022-2024 ASEC H5 inputs consumed by the hermetic --asec-h5 build omit FIN_VAL, FIN_YN, and I_FINVAL from their person tables. HFINVAL and FFINVAL are household/family aggregates and cannot recover the person recipient without synthetic allocation. The retired PUF-clone QRF at datasets/cps/extended_cps.py lines 140-194 and 639-745 trains financial_assistance from CPS person observations, so PUF provides no independent replacement source.", "issue": "PolicyEngine/microcosm#38", diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 0e8c171624..82906e87ba 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -72,9 +72,8 @@ "status": "required" }, "employer_sponsored_insurance_premiums": { - "issue": "PolicyEngine/microcosm#32", - "reason": "SOURCE UNAVAILABILITY WITH EVIDENCE: archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 197-271 and 1575-1581 requires NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL; datasets/cps/census_cps.py lines 13-55 declares the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs consumed by the hermetic --asec-h5 build contain PHIP_VAL but omit all three required NOW_* fields, so the retired owner/payment/plan-type derivation cannot execute. has_esi and NOW_GRP are not semantically valid substitutes.", - "status": "reviewed_exclusion" + "note": "Employer share of the premium for employed current ESI policyholders (PolicyEngine/microcosm#454), written by the meps_esi_premiums base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person files) and MEPS-IC tier x firm-size x State cells, scaled to BEA NIPA 7.8 line 17. A release must persist it: CBO household market income (gov.household.cbo_market_income_additions) and every ESI-to-wages reform bind through it. Required with NO reviewed exclusion; absent on pre-#454 bases by construction — the intended red gate until a base built with the stage ships. The release also holds its total to the anchor (us_esi_premiums_anchor_gate).", + "status": "required" }, "employment_income_before_lsr": { "status": "required" @@ -309,6 +308,10 @@ "pre_subsidy_rent": { "status": "required" }, + "pre_tax_health_insurance_premiums": { + "note": "Employee premium paid through section 125 payroll deduction (PolicyEngine/microcosm#454), written by the same meps_esi_premiums stage: an employed policyholder with wages, a reported premium (PHIP_VAL) and an employee share pays it pre-tax at the MEPS-IC 2025 firm-size pretax-offer rate. PolicyEngine-US 2.2.1 subtracts it from income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, fica_pre_tax_contributions); absent, both wage bases overstate. Required with NO reviewed exclusion; absent on pre-#454 bases by construction.", + "status": "required" + }, "previous_year_income_available": { "status": "required" }, @@ -550,11 +553,11 @@ } }, "counts": { - "required": 167, - "reviewed_exclusion": 7, - "total": 174 + "required": 169, + "reviewed_exclusion": 6, + "total": 175 }, - "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", + "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893), and the employee section 125 premium leaf pre_tax_health_insurance_premiums (the meps_esi_premiums base-builder stage; PolicyEngine/microcosm#454). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", "description": "Declared full-coverage contract for a US release: every input column the reference eCPS exports must be persisted as a key with non-default signal, or carry a reviewed exclusion. Enforced as a hard release gate (microcosm.build.us_runtime.release_input_coverage) that generalizes assert_required_us_release_source_columns from 5 columns to the full eCPS input surface.", "issue": "PolicyEngine/microcosm#368", "reference": { @@ -567,6 +570,38 @@ "vintage": "2024" }, "reform_coverage_probes": [ + { + "binding_inputs": [ + "employer_sponsored_insurance_premiums" + ], + "budget_measure": "cbo_household_market_income", + "effect_direction": "baseline_minus_reform", + "expected_sign": "positive", + "id": "employer_sponsored_insurance_premium_neutralization", + "issue": "PolicyEngine/microcosm#454", + "min_abs_effect": 500000000000.0, + "name": "Employer-sponsored insurance premium neutralization", + "neutralized_variable": "employer_sponsored_insurance_premiums", + "parameter_changes": {}, + "period": 2024, + "reason": "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums to CBO household market income through gov.household.cbo_market_income_additions, its only consumer, so neutralizing it lowers cbo_household_market_income by the weighted employer-premium total: about $977 billion at 2024 (BEA NIPA 7.8 line 17, the meps_esi_premiums anchor; the stage measured $977.0 billion over 74.0 million weighted employed policyholders on the default 2023-2025 pool, before calibration). The floor sits at about half of that. A structural zero means the column was dropped or the stage never ran." + }, + { + "binding_inputs": [ + "pre_tax_health_insurance_premiums" + ], + "budget_measure": "income_tax", + "effect_direction": "baseline_minus_reform", + "expected_sign": "negative", + "id": "pre_tax_health_insurance_premium_neutralization", + "issue": "PolicyEngine/microcosm#454", + "min_abs_effect": 5000000000.0, + "name": "Pre-tax health insurance premium neutralization", + "neutralized_variable": "pre_tax_health_insurance_premiums", + "parameter_changes": {}, + "period": 2024, + "reason": "PolicyEngine-US 2.2.1 subtracts pre_tax_health_insurance_premiums from employment_income in irs_employment_income (through gov.irs.gross_income.pre_tax_contributions), so neutralizing it raises taxable wages and income tax: baseline-minus-reform is negative. The meps_esi_premiums stage measured $181.1 billion of pre-tax premiums over 52.5 million weighted workers on the default 2023-2025 pool; at a typical 10-22 percent marginal rate that is roughly $18-40 billion, so a $5 billion floor separates a structural zero (the column dropped) from any plausible effect." + }, { "binding_inputs": [ "self_employment_income_last_year" diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 72b56c12ab..e2bfce79b8 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1746,55 +1746,132 @@ }, { "stage": "meps_esi_premiums", - "survey": "MEPS-IC", + "survey": "MEPS-IC + CPS ASEC current employment-based coverage", "source": "https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", "grain": "person", "artifacts": [ { "kind": "public_microdata", "format": "census_asec", - "vintage": "hermetic ASEC pool years (2022-2024 for target 2024)", - "locator": "Census CPS ASEC NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL fields; the three NOW_* fields are absent from the current hermetic HDF inputs" + "vintage": "pinned Census ASEC person members, income years 2022-2025", + "locator": "Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM join; every pinned H5 lacks them); PEIO1COW, PHIP_VAL and WSAL_VAL from the pinned H5 inputs; household GESTFIPS as state_fips", + "source_columns": [ + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP", + "PEIO1COW", + "PHIP_VAL", + "WSAL_VAL", + "state_fips" + ] }, { "kind": "administrative_table", - "format": "published_table", - "vintage": "build_year", - "locator": "MEPS Insurance Component plan-type premium tables" + "format": "pdf", + "vintage": "2025", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf", + "sha256": "ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", + "size_bytes": 12530005, + "publication": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee) and II.C/D/E.2 (average employee contribution)" + }, + { + "kind": "administrative_table", + "format": "pdf", + "vintage": "2024", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf", + "sha256": "4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", + "size_bytes": 10687163, + "publication": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025" + }, + { + "kind": "administrative_table", + "format": "pdf", + "vintage": "2024", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_3/2024/ic24_iiia_g.pdf", + "sha256": "462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79", + "size_bytes": 1765717, + "publication": "MEPS-IC 2024 Series III (State and local government, by government type and size and census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released)." + }, + { + "kind": "administrative_table", + "format": "pdf", + "vintage": "2025", + "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_1/2025/ic25_ia_g.pdf", + "sha256": "cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3", + "size_bytes": 9210366, + "publication": "MEPS-IC 2025 Series I (private sector, national): Table I.A.2 (percent of establishments that offer health insurance) and Table I.A.2.j (percent of establishments that offer pretax employee contributions to health insurance), by firm size" + }, + { + "kind": "administrative_table", + "format": "xlsx", + "vintage": "BEA annual update published 2026-09-30", + "locator": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "size_bytes": 944273, + "measure": "NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025", + "publication": "scale anchor: employer contributions for group health insurance as a supplement to wages and salaries" + }, + { + "kind": "administrative_validation_only", + "format": "xlsx", + "vintage": "NHE 2024 release (data through CY2024)", + "locator": "https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx", + "measure": "employer and employee contributions to employer-sponsored private health insurance premiums (upper bounds: NHEA includes COBRA and retiree enrollees)" } ], "operations": [ { "kind": "read_table", - "table": "meps_ic_plan_premiums" + "table": "person", + "weight": "person_weight" }, { - "kind": "assign_by_plan_type", - "inputs": [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - "PHIP_VAL" + "kind": "derive_employer_sponsored_insurance_premiums", + "cells_resource": "microcosm.build.us_runtime.data/meps_ic_esi_premium_cells.json", + "cells_sha256": "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8", + "universe": "NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6)", + "tier_column": "NOW_GRPFTYP2", + "size_column": "NOEMP", + "sector_column": "PEIO1COW", + "payment_column": "NOW_HIPAID", + "payment_rule": "1 (employer paid all): average total premium; 2 (some): average total premium minus average employee contribution; 3 (none): 0", + "suppressed_cell_fallback": "national firm-size cell times the State total-to-national ratio", + "government_cell_aging": "MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same tier and measure", + "anchor": { + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "series": "B4923C", + "values": { + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 + } + } + }, + { + "kind": "derive_pre_tax_health_insurance_premiums", + "seed_from_build_config": true, + "premium_column": "PHIP_VAL", + "wage_column": "WSAL_VAL", + "employer_payment_codes": [ + 2, + 3 ], - "owner_code": 1, - "employer_pays_all_code": 1, - "employer_pays_some_code": 2, - "family_plan_code": 1, - "self_only_plan_code": 2, - "source_unavailable_when_missing": [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP" - ] + "pre_tax_share": "MEPS-IC 2025 Table I.A.2.j pretax-contribution offer percent / Table I.A.2 health-insurance offer percent, by firm size" } ], "outputs": [ - "employer_sponsored_insurance_premiums" + "employer_sponsored_insurance_premiums", + "pre_tax_health_insurance_premiums" ], "nonnegative_outputs": [ - "employer_sponsored_insurance_premiums" + "employer_sponsored_insurance_premiums", + "pre_tax_health_insurance_premiums" ], - "notes": "The retired derivation identifies the policy owner with NOW_OWNGRP, employer payment status with NOW_HIPAID, family/self-only plan type with NOW_GRPFTYP, and reported employee-paid premium with PHIP_VAL, then applies MEPS-IC family/self-only total and employee contribution priors (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/cps/cps.py lines 197-271 and 1575-1581). datasets/cps/census_cps.py lines 13-55 declares the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs used by the hermetic build omit the three NOW_* fields, so this stage is intentionally source-unavailable; has_esi and NOW_GRP are forbidden proxies." + "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Universe: employed (PEMLR 1-2) current ESI policyholders (NOW_OWNGRP 1) whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders (retiree and COBRA coverage is not compensation of a current job), the unincorporated self-employed and workers without pay carry 0. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium; some (2): premium minus contribution; none (3): 0. One factor scales every share so the weighted total equals BEA NIPA 7.8 line 17 for the build year ($977.0B CY2024 in the 2026-09-30 vintage; the retired us-data target was the earlier $1,002.9B vintage). NHE Table 24 ($1,047.0B CY2024, banked as cms_nhe.cy2024.esi_employer_contribution_premiums) counts COBRA and retiree enrollees and is an upper bound; applying the same cells to every current policyholder reproduces it within 2 percent on both pinned pools ($1,031.0B on income years 2023-2025, $1,036.8B on 2022-2024), non-employed and no-employer policyholders carrying 11.5 percent (experiments/us-esi-454). pre_tax_health_insurance_premiums: an employed policyholder with wages (WSAL_VAL > 0), a reported premium (PHIP_VAL > 0) and an employee share (NOW_HIPAID 2-3) pays PHIP_VAL through payroll with the firm-size probability that an insurance-offering employer offers pretax contributions (MEPS-IC 2025 Table I.A.2.j / Table I.A.2: 49.5 percent under 50, 92.6 percent 50+, 72.2 percent all sizes); establishment-weighted, so a lower estimate. Seeded blake2b draws keyed by source identity; support clones agree. PolicyEngine-US 2.2.1 subtracts it from income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, fica_pre_tax_contributions). Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples) and us_esi_premiums_anchor_gate (calibrated total within 5 percent of BEA, at most NHE)." }, { "stage": "prior_year_income", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index e33ffec848..6877c0b5a7 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf + sha256: 0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91 stage_manifest: version: 1 country: us @@ -1783,48 +1783,144 @@ stages: per employer (inherited upward concept edge). Adjudication and measurements: microcosm#451 item 4 overtime-incidence lane.' - stage: meps_esi_premiums - survey: MEPS-IC + survey: MEPS-IC + CPS ASEC current employment-based coverage source: https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp grain: person artifacts: - kind: public_microdata format: census_asec - vintage: hermetic ASEC pool years (2022-2024 for target 2024) - locator: Census CPS ASEC NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, and PHIP_VAL fields; the three NOW_* - fields are absent from the current hermetic HDF inputs - - kind: administrative_table - format: published_table - vintage: build_year - locator: MEPS Insurance Component plan-type premium tables - operations: - - kind: read_table - table: meps_ic_plan_premiums - - kind: assign_by_plan_type - inputs: + vintage: pinned Census ASEC person members, income years 2022-2025 + locator: Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and + NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM + join; every pinned H5 lacks them); PEIO1COW, PHIP_VAL and WSAL_VAL from the pinned H5 inputs; household + GESTFIPS as state_fips + source_columns: - NOW_OWNGRP - NOW_HIPAID - NOW_GRPFTYP + - NOW_GRPFTYP2 + - PEMLR + - NOEMP + - PEIO1COW - PHIP_VAL - owner_code: 1 - employer_pays_all_code: 1 - employer_pays_some_code: 2 - family_plan_code: 1 - self_only_plan_code: 2 - source_unavailable_when_missing: - - NOW_OWNGRP - - NOW_HIPAID - - NOW_GRPFTYP + - WSAL_VAL + - state_fips + - kind: administrative_table + format: pdf + vintage: '2025' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf + sha256: ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330 + size_bytes: 12530005 + publication: 'MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average + total premium per enrolled employee) and II.C/D/E.2 (average employee contribution)' + - kind: administrative_table + format: pdf + vintage: '2024' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf + sha256: 4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b + size_bytes: 10687163 + publication: MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows + age the 2024 State and local government cells to 2025 + - kind: administrative_table + format: pdf + vintage: '2024' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_3/2024/ic24_iiia_g.pdf + sha256: 462afdd2b2629b32b0c4822a33d7e32260e4c4e196f6098f0d933a3c8ba0be79 + size_bytes: 1765717 + publication: 'MEPS-IC 2024 Series III (State and local government, by government type and size and + census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this + series (the 2025 series is not yet released).' + - kind: administrative_table + format: pdf + vintage: '2025' + locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_1/2025/ic25_ia_g.pdf + sha256: cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3 + size_bytes: 9210366 + publication: 'MEPS-IC 2025 Series I (private sector, national): Table I.A.2 (percent of establishments + that offer health insurance) and Table I.A.2.j (percent of establishments that offer pretax employee + contributions to health insurance), by firm size' + - kind: administrative_table + format: xlsx + vintage: BEA annual update published 2026-09-30 + locator: https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx + sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd + size_bytes: 944273 + measure: NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025 + publication: 'scale anchor: employer contributions for group health insurance as a supplement to wages + and salaries' + - kind: administrative_validation_only + format: xlsx + vintage: NHE 2024 release (data through CY2024) + locator: https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx + measure: 'employer and employee contributions to employer-sponsored private health insurance premiums + (upper bounds: NHEA includes COBRA and retiree enrollees)' + operations: + - kind: read_table + table: person + weight: person_weight + - kind: derive_employer_sponsored_insurance_premiums + cells_resource: microcosm.build.us_runtime.data/meps_ic_esi_premium_cells.json + cells_sha256: fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8 + universe: NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6) + tier_column: NOW_GRPFTYP2 + size_column: NOEMP + sector_column: PEIO1COW + payment_column: NOW_HIPAID + payment_rule: '1 (employer paid all): average total premium; 2 (some): average total premium minus + average employee contribution; 3 (none): 0' + suppressed_cell_fallback: national firm-size cell times the State total-to-national ratio + government_cell_aging: MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same + tier and measure + anchor: + source: https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx + sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd + series: B4923C + values: + '2023': 923195000000 + '2024': 977034000000 + '2025': 1027929000000 + - kind: derive_pre_tax_health_insurance_premiums + seed_from_build_config: true + premium_column: PHIP_VAL + wage_column: WSAL_VAL + employer_payment_codes: + - 2 + - 3 + pre_tax_share: MEPS-IC 2025 Table I.A.2.j pretax-contribution offer percent / Table I.A.2 health-insurance + offer percent, by firm size outputs: - employer_sponsored_insurance_premiums + - pre_tax_health_insurance_premiums nonnegative_outputs: - employer_sponsored_insurance_premiums - notes: The retired derivation identifies the policy owner with NOW_OWNGRP, employer payment status with - NOW_HIPAID, family/self-only plan type with NOW_GRPFTYP, and reported employee-paid premium with PHIP_VAL, - then applies MEPS-IC family/self-only total and employee contribution priors (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, - datasets/cps/cps.py lines 197-271 and 1575-1581). datasets/cps/census_cps.py lines 13-55 declares - the three NOW_* fields optional. All three SHA-recorded 2022-2024 HDF inputs used by the hermetic - build omit the three NOW_* fields, so this stage is intentionally source-unavailable; has_esi and - NOW_GRP are forbidden proxies. + - pre_tax_health_insurance_premiums + notes: 'Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) + from source fields the hermetic H5 inputs lacked (microcosm#454). Universe: employed (PEMLR 1-2) current + ESI policyholders (NOW_OWNGRP 1) whose current job has an employer (PEIO1COW 1-6); dependents, non-employed + policyholders (retiree and COBRA coverage is not compensation of a current job), the unincorporated + self-employed and workers without pay carry 0. Cells: MEPS-IC 2025 Series II private-sector average + total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under + 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and + coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); + State government and local/federal employers take the MEPS-IC 2024 Series III State-government and + all-government census-division cells (latest published), aged to 2025 by the private national 2025/2024 + ratio of the same tier and measure. AHRQ suppressed nine small-firm contribution cells; each takes + the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID + 1): premium; some (2): premium minus contribution; none (3): 0. One factor scales every share so the + weighted total equals BEA NIPA 7.8 line 17 for the build year ($977.0B CY2024 in the 2026-09-30 vintage; + the retired us-data target was the earlier $1,002.9B vintage). NHE Table 24 ($1,047.0B CY2024, banked + as cms_nhe.cy2024.esi_employer_contribution_premiums) counts COBRA and retiree enrollees and is an + upper bound; applying the same cells to every current policyholder reproduces it within 2 percent + on both pinned pools ($1,031.0B on income years 2023-2025, $1,036.8B on 2022-2024), non-employed and + no-employer policyholders carrying 11.5 percent (experiments/us-esi-454). pre_tax_health_insurance_premiums: + an employed policyholder with wages (WSAL_VAL > 0), a reported premium (PHIP_VAL > 0) and an employee + share (NOW_HIPAID 2-3) pays PHIP_VAL through payroll with the firm-size probability that an insurance-offering + employer offers pretax contributions (MEPS-IC 2025 Table I.A.2.j / Table I.A.2: 49.5 percent under + 50, 92.6 percent 50+, 72.2 percent all sizes); establishment-weighted, so a lower estimate. Seeded + blake2b draws keyed by source identity; support clones agree. PolicyEngine-US 2.2.1 subtracts it from + income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, fica_pre_tax_contributions). + Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples) and us_esi_premiums_anchor_gate + (calibrated total within 5 percent of BEA, at most NHE).' - stage: prior_year_income survey: CPS ASEC (prior year) source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index 1ffcd46431..37502a9688 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -322,6 +322,10 @@ seed_site_bindings: owners: - kind: source_stage id: pregnancy +- site: pre_tax_health_premium_assignment + owners: + - kind: source_stage + id: meps_esi_premiums - site: wic_claim_assignment owners: - kind: source_stage diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 904f8bf512..743f2d579d 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -9,7 +9,7 @@ "family_granularity": "namespace.concept of the ledger source_record_id (us_target_family_id)", "governing_rule": "us-data-targeted => compiled unless source-absent; every reviewed exclusion carries a Chesterton's-fence {origin, purpose, verdict_basis}", "compiled_families": "32", - "reviewed_exclusions": "52" + "reviewed_exclusions": "53" }, "families": { "bea_nipa.business_current_transfer_receipts": { @@ -232,6 +232,17 @@ "verdict_basis": "no fence to rebuild. Programs some of these NIPA aggregates measure (Social Security, Medicare, Medicaid, unemployment) are calibrated from SSA/CMS/CBO admin sources (compiled), not the BEA NIPA series." } }, + "bea_nipa.private_group_health_insurance": { + "status": "reviewed_exclusion", + "classification": "source_absent", + "reason": "BEA NIPA Table 7.8 line 17 private group health insurance (B4923C), the employer-contribution aggregate the retired pipeline calibrated employer_sponsored_insurance_premiums to (loss.py 1_002.9e9). The pinned feed carries no fact for it.", + "evidence": "retired us-data pipeline (archived 2e044571) loss.py line 33; feed bea_nipa families carry no group-health series", + "fence": { + "origin": "us-data loss.py employer_sponsored_insurance_premiums target", + "purpose": "anchor employer-paid ESI premiums for current employees: the compensation concept the employed-policyholder column carries.", + "verdict_basis": "source-absent from the feed: the meps_esi_premiums stage pins the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., published 2026-09-30: $977.034B CY2024) and scales to it, and the release anchor gate validates against it; compile it here once the ledger banks the series." + } + }, "bea_nipa.proprietors_income": { "status": "compiled" }, @@ -448,24 +459,24 @@ }, "cms_nhe.esi_employer_contribution_premiums": { "status": "reviewed_exclusion", - "classification": "not_modeled", - "reason": "NHE employer-contribution ESI premium aggregate: a retired us-data calibration target (loss.py 1_002.9e9, removed at 9f320d2c) whose anchor v9.2 re-banks (2026-07-21) for microcosm#454's future gate. The pe-us input it would anchor (employer_sponsored_insurance_premiums, the employer-paid share per the variable's own documentation) has no live producer in the hermetic lineage, so a target would bind against a structural zero. The all-employer series is the future anchor (pe-us covers government and private employees alike); the private-employer subset rides as its cross-check.", - "evidence": "ecps_parity_known_gaps.json entry employer_sponsored_insurance_premiums (NOW_* source-unavailability evidence); microcosm#454 descope comment 2026-07-21", + "classification": "deferred", + "reason": "NHE Table 24 employer contribution to ESI premiums. The pe-us input it anchors (employer_sponsored_insurance_premiums) is now produced by the meps_esi_premiums base stage (microcosm#454) for employed current ESI policyholders only, while NHE counts COBRA and retiree enrollees (NHEA methodology paper 2024, p. 28; Table 24 note 1 adds Medicare Retiree Drug Subsidy payments), so a calibration target on this fact would load the retiree share onto workers. Validation-gated instead: us_esi_premiums_anchor_gate holds the calibrated total within 5% of BEA NIPA 7.8 line 17 (the us-data target's concept, pinned in the stage) and at or below this all-employer fact; the private-employer subset rides as the sector cross-check.", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: MEPS-IC cells over all 92.4M weighted current policyholders give $1,031.0B (0.985 x NHE $1,047.0B CY2024); the employed universe gives $912.8B raw, scaled to BEA $977.0B (11.5% of the all-policyholder total is non-employed or no-employer coverage); private employers $756.7B vs NHE private $753.4B", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed facts re-bank that same anchor for the future gate, entered deliberately ahead of the producer.", - "purpose": "anchor a validation gate for the CBO market-income ESI component once the column has a live producer: NIPA 7.8 employer contributions for group health insurance and the NHEA sponsor-of-funds employer share are same-concept estimates (~4 percent apart, CY2024).", - "verdict_basis": "us-data-targeted, so under the absolute rule this compiles UNLESS source-absent \u2014 and the producer source is absent: the declared meps_esi_premiums stage cannot execute because all three required NOW_* fields are missing from the sha-pinned 2022-2024 ASEC h5 inputs (recorded parity known-gap), so a target would bind against a structural zero. Compiles when microcosm#454 restores the source fields or lands a validated PHIP_VAL+MEPS re-derivation; the anchor value is already banked in the feed." + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", + "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NIPA 7.8 line 17 counts employer contributions as compensation of current employees (BEA builds it from MEPS-IC 'insurance purchased by employers for employees'); NHE Table 24 adds COBRA and retiree enrollees, so it bounds the column from above (7.2% above BEA in CY2024).", + "verdict_basis": "deferred, not dropped: the producer is live, but the only banked fact measures a broader concept than the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) uses it as the upper bound today; bank BEA NIPA 7.8 line 17 (B4923C) in the ledger and compile that as the calibration target, and this family stays the cross-check." } }, "cms_nhe.esi_private_employer_contribution_premiums": { "status": "reviewed_exclusion", - "classification": "not_modeled", - "reason": "NHE employer-contribution ESI premium aggregate: a retired us-data calibration target (loss.py 1_002.9e9, removed at 9f320d2c) whose anchor v9.2 re-banks (2026-07-21) for microcosm#454's future gate. The pe-us input it would anchor (employer_sponsored_insurance_premiums, the employer-paid share per the variable's own documentation) has no live producer in the hermetic lineage, so a target would bind against a structural zero. The all-employer series is the future anchor (pe-us covers government and private employees alike); the private-employer subset rides as its cross-check.", - "evidence": "ecps_parity_known_gaps.json entry employer_sponsored_insurance_premiums (NOW_* source-unavailability evidence); microcosm#454 descope comment 2026-07-21", + "classification": "deferred", + "reason": "NHE Table 24 employer contribution to ESI premiums. The pe-us input it anchors (employer_sponsored_insurance_premiums) is now produced by the meps_esi_premiums base stage (microcosm#454) for employed current ESI policyholders only, while NHE counts COBRA and retiree enrollees (NHEA methodology paper 2024, p. 28; Table 24 note 1 adds Medicare Retiree Drug Subsidy payments), so a calibration target on this fact would load the retiree share onto workers. Validation-gated instead: us_esi_premiums_anchor_gate holds the calibrated total within 5% of BEA NIPA 7.8 line 17 (the us-data target's concept, pinned in the stage) and at or below this all-employer fact; the private-employer subset rides as the sector cross-check.", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: MEPS-IC cells over all 92.4M weighted current policyholders give $1,031.0B (0.985 x NHE $1,047.0B CY2024); the employed universe gives $912.8B raw, scaled to BEA $977.0B (11.5% of the all-policyholder total is non-employed or no-employer coverage); private employers $756.7B vs NHE private $753.4B", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed facts re-bank that same anchor for the future gate, entered deliberately ahead of the producer.", - "purpose": "anchor a validation gate for the CBO market-income ESI component once the column has a live producer: NIPA 7.8 employer contributions for group health insurance and the NHEA sponsor-of-funds employer share are same-concept estimates (~4 percent apart, CY2024).", - "verdict_basis": "us-data-targeted, so under the absolute rule this compiles UNLESS source-absent \u2014 and the producer source is absent: the declared meps_esi_premiums stage cannot execute because all three required NOW_* fields are missing from the sha-pinned 2022-2024 ASEC h5 inputs (recorded parity known-gap), so a target would bind against a structural zero. Compiles when microcosm#454 restores the source fields or lands a validated PHIP_VAL+MEPS re-derivation; the anchor value is already banked in the feed." + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", + "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NIPA 7.8 line 17 counts employer contributions as compensation of current employees (BEA builds it from MEPS-IC 'insurance purchased by employers for employees'); NHE Table 24 adds COBRA and retiree enrollees, so it bounds the column from above (7.2% above BEA in CY2024).", + "verdict_basis": "deferred, not dropped: the producer is live, but the only banked fact measures a broader concept than the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) uses it as the upper bound today; bank BEA NIPA 7.8 line 17 (B4923C) in the ledger and compile that as the calibration target, and this family stays the cross-check." } }, "cms_nhe.medicaid_title_xix_expenditures": { diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 74e4757cba..10cd7c5adf 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -265,6 +265,22 @@ us_energy_subsidy_summary, with_us_energy_subsidy_input, ) +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PRE_TAX_PREMIUM_COLUMN, + US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, + US_ESI_PREMIUMS_OUTPUT_COLUMNS, + US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, + US_ESI_PREMIUMS_STAGE_NAME, + derive_us_employer_esi_premiums_from_manifest, + derive_us_pre_tax_health_insurance_premiums_from_manifest, + load_meps_ic_esi_premium_cells, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + us_esi_premiums_stage_spec, + us_esi_premiums_summary, + with_us_esi_premium_inputs, +) from microcosm.build.us_runtime.farm_business_income import ( FARM_BUSINESS_INCOME_ARCHIVED_CPS_FARM_INCOME_URL, FARM_BUSINESS_INCOME_ARCHIVED_DERIVATION_URL, @@ -1351,6 +1367,20 @@ "us_energy_subsidy_stage_spec", "us_energy_subsidy_summary", "with_us_energy_subsidy_input", + "US_ESI_EMPLOYER_PREMIUM_COLUMN", + "US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS", + "US_ESI_PREMIUMS_OUTPUT_COLUMNS", + "US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS", + "US_ESI_PREMIUMS_STAGE_NAME", + "US_ESI_PRE_TAX_PREMIUM_COLUMN", + "derive_us_employer_esi_premiums_from_manifest", + "derive_us_pre_tax_health_insurance_premiums_from_manifest", + "load_meps_ic_esi_premium_cells", + "us_esi_premiums_anchor_gate", + "us_esi_premiums_signal_gate", + "us_esi_premiums_stage_spec", + "us_esi_premiums_summary", + "with_us_esi_premium_inputs", "CHILD_SUPPORT_ARCHIVED_PUF_IMPUTATION_URL", "CHILD_SUPPORT_ARCHIVED_PUF_OUTPUTS_URL", "CHILD_SUPPORT_EXPENSE_ARCHIVED_DERIVATION_URL", @@ -2135,10 +2165,16 @@ def to_manifest(self) -> dict[str, object]: "structurally impossible under StagePlan." ), ), - "meps_esi_premiums": DonorSpec( - survey="MEPS-IC", + US_ESI_PREMIUMS_STAGE_NAME: DonorSpec( + survey="MEPS-IC + CPS ASEC current employment-based coverage", source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", - notes="Employer-sponsored insurance premium parameters.", + notes=( + "Employed current ESI policyholders take the MEPS-IC employer " + "share of their tier x firm-size x State (or government " + "census-division) cell, scaled to BEA NIPA 7.8 line 17; reported " + "premiums route through payroll at MEPS-IC pretax-offer rates " + "(microcosm #454). Shared by support clones." + ), ), "aca_marketplace_inputs": DonorSpec( survey="CPS ASEC + CMS Marketplace Open Enrollment PUFs", @@ -2501,7 +2537,7 @@ def to_manifest(self) -> dict[str, object]: US_SSI_TAKE_UP_STAGE_NAME, "sipp_tips", "org_wages", - "meps_esi_premiums", + US_ESI_PREMIUMS_STAGE_NAME, "mortgage_conversion", "vehicle_assets", US_VOLUNTARY_FILING_STAGE_NAME, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py index 665789e6b6..d0187f65d3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_census_person_columns.py @@ -61,6 +61,27 @@ The other thirteen columns the 2026-08-23 offline fix appended are not restored; :data:`ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED` records why. + +Microcosm #454 adds six columns that every pinned H5 lacks (2024 included), +read by the ``meps_esi_premiums`` stage (:mod:`.esi_premiums`): + +====================== ====================================================== +Column Build reader +====================== ====================================================== +``NOW_OWNGRP`` ``esi_premiums._esi_person_codes`` -> the current + employment-based policyholder test +``NOW_HIPAID`` same -> employer paid all / some / none of the premium +``NOW_GRPFTYP2`` same -> coverage tier (family / self plus one / + self-only) +``NOW_GRPFTYP`` same -> cross-checked against ``NOW_GRPFTYP2`` +``PEMLR`` same -> employed at interview +``NOEMP`` same -> employer size band +====================== ====================================================== + +Their domains are the codes observed in pppub23/24/25/26 (2026-10-10), equal +to the Census API codebooks for survey years 2023-2026 except that the data +also carry ``PEMLR == 0`` (not in universe: children and Armed Forces), which +the codebooks omit. """ from __future__ import annotations @@ -107,6 +128,8 @@ class AsecCensusPersonColumn: _NOW_YES_NO = frozenset({1, 2}) _ENROLLMENT = frozenset({0, 1, 2}) +_ESI_READER = "microcosm.build.us_runtime.esi_premiums._esi_person_codes" + _HEALTH_COVERAGE_READER = ( "microcosm.build.us_runtime.cps_carried._fill_health_coverage_inputs" ) @@ -118,6 +141,7 @@ class AsecCensusPersonColumn: #: the Census codes observed in the pinned person members pppub23/24/25 #: (2026-09-23); pppub26 (income year 2025) falls inside every one of them #: (2026-09-27). ``A_EXPRRP`` has no code 6 in the Census codebook or the data. +#: The six #454 columns were observed in pppub23/24/25/26 on 2026-10-10. ASEC_CENSUS_PERSON_COLUMNS: tuple[AsecCensusPersonColumn, ...] = ( AsecCensusPersonColumn( "NOW_MCAID", @@ -205,6 +229,38 @@ class AsecCensusPersonColumn: "(is_full_time_college_student <- A_HSCOL == 2 & A_FTPT == 1)", ), ), + # Microcosm #454: the meps_esi_premiums stage's universe, tier, payment + # status, employment and employer-size fields. + AsecCensusPersonColumn( + "NOW_OWNGRP", + frozenset({0, 1, 2}), + (f"{_ESI_READER} (current employment-based policyholder, 1 = yes)",), + ), + AsecCensusPersonColumn( + "NOW_HIPAID", + frozenset({0, 1, 2, 3}), + (f"{_ESI_READER} (employer paid all / some / none of the premium)",), + ), + AsecCensusPersonColumn( + "NOW_GRPFTYP", + frozenset({0, 1, 2}), + (f"{_ESI_READER} (family / self-only, checked against NOW_GRPFTYP2)",), + ), + AsecCensusPersonColumn( + "NOW_GRPFTYP2", + frozenset({0, 1, 2, 3}), + (f"{_ESI_READER} (family / self plus one / self-only coverage tier)",), + ), + AsecCensusPersonColumn( + "PEMLR", + frozenset(range(8)), + (f"{_ESI_READER} (employed at interview: codes 1 and 2)",), + ), + AsecCensusPersonColumn( + "NOEMP", + frozenset(range(7)), + (f"{_ESI_READER} (employer size: under 50 = codes 1-2, 50+ = 3-6)",), + ), ) ASEC_CENSUS_PERSON_COLUMN_NAMES: tuple[str, ...] = tuple( column.name for column in ASEC_CENSUS_PERSON_COLUMNS diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py new file mode 100644 index 0000000000..4c1960ff94 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py @@ -0,0 +1,1112 @@ +"""MEPS-IC employer premiums and pre-tax employee premiums for ESI policyholders. + +Microcosm #454. PolicyEngine-US 2.2.1 reads two person inputs that no +Microcosm stage produced: + +* ``employer_sponsored_insurance_premiums`` — "annual employer-paid health + insurance premiums", one of ``gov.household.cbo_market_income_additions`` + (CBO household market income); and +* ``pre_tax_health_insurance_premiums`` — premiums paid through pre-tax + payroll deductions, listed in ``gov.irs.gross_income.pre_tax_contributions`` + (subtracted from ``employment_income`` in ``irs_employment_income``) and + ``gov.irs.gross_income.fica_pre_tax_contributions`` (subtracted in + ``payroll_tax_gross_wages``). + +The retired eCPS derivation (archived 42ed5d45 ``datasets/cps/cps.py`` +L197-271) read the CPS ASEC current employment-based coverage fields, which the +pinned H5 inputs omit; :mod:`.asec_census_person_columns` now restores them, +with employment status and employer size, from the pinned Census person files. + +Employer premium +---------------- +The universe is the **employed** (``PEMLR`` 1-2) **current ESI policyholder** +(``NOW_OWNGRP`` 1). Dependents carry nothing (the policyholder carries the +premium) and non-employed policyholders carry nothing (retiree and COBRA +coverage is not compensation of a current job). Self-employed-unincorporated +workers and workers without pay (``PEIO1COW`` 7-8) carry nothing either: their +current job has no employer to pay a share. + +Each policyholder's raw employer share comes from the MEPS-IC cell of their +coverage tier (``NOW_GRPFTYP2``: family, self plus one, self-only), employer +sector (``PEIO1COW``) and, for private employers, State and firm size +(``NOEMP`` under 50 / 50 or more): + +* employer paid all of the premium (``NOW_HIPAID`` 1): the cell's average + total premium; +* employer paid some (2): the average total premium minus the average employee + contribution; +* employer paid none (3): zero. + +Private-sector cells are MEPS-IC 2025 Series II (State by firm size). +Government cells are MEPS-IC 2024 Series III by census division (the latest +year published), aged to 2025 by the private-sector national 2025/2024 ratio of +the same tier and measure: State government employees take the +State-government column, local and federal employees the all-governments +column (MEPS-IC does not survey the federal government, so that is a +stand-in). AHRQ suppresses nine small-firm employee-contribution cells; each +takes the national small-firm cell times the State's total-to-national ratio. + +A single factor then scales every raw share so the weighted total equals BEA +NIPA Table 7.8 line 17 (series B4923C, "Private group health insurance", +employer contributions as a supplement to wages and salaries; government +employers' contributions to privately administered plans included) for the +build year. BEA's NIPA handbook (chapter 10, table 10.B) says the private and +State and local parts of that series come from MEPS data "on insurance +purchased by employers for employees", the active-employee concept this column +carries. CMS NHE Table 24's employer contribution is the cross-check, and an +upper bound: the NHEA methodology counts premiums for active employees, COBRA +enrollees and retirees. + +Pre-tax employee premium +------------------------ +Eligible: an employed policyholder with wages last year (``WSAL_VAL`` > 0), a +positive reported premium (``PHIP_VAL``) and an employee share to pay +(``NOW_HIPAID`` 2 or 3). An eligible person pays the reported premium pre-tax +with the firm-size probability that their employer offers pretax employee +contributions, among employers that offer health insurance: MEPS-IC 2025 Table +I.A.2.j over Table I.A.2. The rates are establishment-weighted; the published +bands rise with firm size (8.9% under 10 employees, 92.6% at 1,000 or more), +so they understate the enrollee-weighted rate and the column is a lower +estimate. +The draw is a seeded blake2b uniform keyed by stable source identity, so +support clones agree. Others carry zero. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass +from importlib.resources import files +from types import MappingProxyType +from typing import Any + +import numpy as np +import pandas as pd + +from microcosm.build.gates import GateResult +from microcosm.build.source_manifest import ( + SourceOperationSpec, + SourceStageSpec, + load_source_manifest, +) +from microcosm.build.source_runtime import ( + SourceRuntimeConfig, + SourceRuntimeContext, + SourceRuntimeError, + run_source_stage, +) +from microcosm.frame import Frame +from microcosm.frame.units import US_SCHEMA + +__all__ = [ + "US_ESI_EMPLOYER_PREMIUM_COLUMN", + "US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS", + "US_ESI_PREMIUMS_OUTPUT_COLUMNS", + "US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS", + "US_ESI_PREMIUMS_STAGE_NAME", + "US_ESI_PRE_TAX_PREMIUM_COLUMN", + "derive_us_employer_esi_premiums_from_manifest", + "derive_us_pre_tax_health_insurance_premiums_from_manifest", + "load_meps_ic_esi_premium_cells", + "us_esi_premiums_anchor_gate", + "us_esi_premiums_signal_gate", + "us_esi_premiums_stage_spec", + "us_esi_premiums_summary", + "with_us_esi_premium_inputs", +] + +US_ESI_PREMIUMS_STAGE_NAME = "meps_esi_premiums" +US_ESI_EMPLOYER_PREMIUM_COLUMN = "employer_sponsored_insurance_premiums" +US_ESI_PRE_TAX_PREMIUM_COLUMN = "pre_tax_health_insurance_premiums" +US_ESI_PREMIUMS_OUTPUT_COLUMNS: tuple[str, ...] = ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PRE_TAX_PREMIUM_COLUMN, +) +US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS = US_ESI_PREMIUMS_OUTPUT_COLUMNS + +#: Raw CPS ASEC person columns the stage reads (the six NOW_*/PEMLR/NOEMP +#: columns through :mod:`.asec_census_person_columns`), plus the household +#: State (``state_fips``, from GESTFIPS), joined onto the person table by +#: ``person_household_id`` when it lives on the household table. +US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS: tuple[str, ...] = ( + "NOW_OWNGRP", + "NOW_HIPAID", + "NOW_GRPFTYP", + "NOW_GRPFTYP2", + "PEMLR", + "NOEMP", + "PEIO1COW", + "PHIP_VAL", + "WSAL_VAL", + "state_fips", +) + +#: Census CPS ASEC person codes (survey years 2023-2026 codebooks, matching the +#: pinned person members on 2026-10-10). +_CODE_DOMAINS: Mapping[str, frozenset[int]] = MappingProxyType( + { + "NOW_OWNGRP": frozenset({0, 1, 2}), + "NOW_HIPAID": frozenset({0, 1, 2, 3}), + "NOW_GRPFTYP": frozenset({0, 1, 2}), + "NOW_GRPFTYP2": frozenset({0, 1, 2, 3}), + "PEMLR": frozenset(range(8)), + "NOEMP": frozenset(range(7)), + "PEIO1COW": frozenset(range(9)), + } +) +_POLICYHOLDER = 1 +_EMPLOYED = (1, 2) +_PAYS_ALL, _PAYS_SOME, _PAYS_NONE = 1, 2, 3 +#: NOW_GRPFTYP2 -> MEPS-IC tier; NOW_GRPFTYP must agree (1 family, 2 self-only). +_TIER_BY_CODE: Mapping[int, str] = MappingProxyType( + {1: "family", 2: "employee_plus_one", 3: "single"} +) +_GRPFTYP_FOR_GRPFTYP2: Mapping[int, int] = MappingProxyType({1: 1, 2: 1, 3: 2}) +#: NOEMP ("Work experience, persons who work for employer"): 1 under 10 and +#: 2 10-49 -> under 50; 3-6 -> 50 or more; 0 (not in universe) -> the State's +#: all-sizes cell. +_SIZE_BY_NOEMP: Mapping[int, str] = MappingProxyType( + { + 0: "total", + 1: "lt50", + 2: "lt50", + 3: "50plus", + 4: "50plus", + 5: "50plus", + 6: "50plus", + } +) +#: PEIO1COW: 1 federal, 2 state, 3 local, 4 private for profit, 5 private +#: nonprofit, 6 self-employed incorporated, 7 self-employed unincorporated, +#: 8 without pay. +_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT = 1, 2, 3 +_PRIVATE_EMPLOYER_CODES = (4, 5, 6) +_EMPLOYER_SECTOR_CODES = (_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT, 4, 5, 6) + +_CELLS_RESOURCE = "meps_ic_esi_premium_cells.json" +_CELLS_SHA256 = "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8" +_PERSON_WEIGHT_COLUMN = "person_weight" +_DRAW_SALT = US_ESI_PRE_TAX_PREMIUM_COLUMN + +_BEA_SECTION7_URL = ( + "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx" +) +_NHE_TABLES_URL = "https://www.cms.gov/files/zip/nhe-tables.zip" + +#: BEA NIPA Table 7.8 line 17 (B4923C), millions of dollars converted to +#: dollars, from the annual update published 2026-09-30 (file created +#: 2026-09-28). That update revised 2024 from the $1,002.9B the retired +#: us-data loss matrix pinned. +EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( + { + "source": _BEA_SECTION7_URL, + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "line": 17, + "series": "B4923C", + "label": "Private group health insurance", + "published": "2026-09-30", + # Integer dollars: the manifest round-trips through YAML. + "values": { + "2023": 923_195_000_000, + "2024": 977_034_000_000, + "2025": 1_027_929_000_000, + }, + } +) +#: CMS NHE Table 24 (2024 release, data through CY2024), the cross-check and +#: upper bound; banked in the pinned feed as +#: cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums and +#: ..._private_employer_contribution_premiums. +EMPLOYER_PREMIUM_CROSS_CHECK: Mapping[str, Any] = MappingProxyType( + { + "source": _NHE_TABLES_URL, + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "table": "Table 24 Employer-Sponsored Private Health Insurance", + "employer_contribution": {"2023": 975.7e9, "2024": 1_047.0e9}, + "employee_contribution": {"2023": 357.8e9, "2024": 382.1e9}, + "relation": ( + "upper bound: NHEA counts employer contributions for active " + "employees, COBRA enrollees and retirees" + ), + } +) +#: Tolerance of the calibrated release aggregate around the BEA anchor. The +#: stage pins the pre-calibration aggregate exactly; the band absorbs +#: calibration and sparse-selection drift. It is wider than BEA's own 2024 +#: revision (2.6%) and narrower than the NHE-BEA gap (7.2%), so a release that +#: drifts toward the retiree-inclusive NHE concept fails. +ANCHOR_RELATIVE_TOLERANCE = 0.05 +#: Weighted-raw employer share per positive holder, before scaling. MEPS-IC +#: 2025 employer shares run $7.2k (single) to $19.0k (family) and the pinned +#: pools measure $12.3-12.4k; this band catches a broken cell table, which the +#: scale factor would otherwise hide. +_RAW_MEAN_BAND = (6_000.0, 20_000.0) +#: Weighted share of all persons with a positive employer premium. Measured +#: on both pinned pools: 22.7%. +_POSITIVE_SHARE_BAND = (0.15, 0.32) +#: Weighted share of all persons with a positive pre-tax premium. Measured on +#: both pinned pools: 16.1%. +_PRE_TAX_POSITIVE_SHARE_BAND = (0.05, 0.25) + +_EMPLOYER_PREMIUM_PARAMETERS: Mapping[str, Any] = MappingProxyType( + { + "cells_resource": f"microcosm.build.us_runtime.data/{_CELLS_RESOURCE}", + "cells_sha256": _CELLS_SHA256, + "universe": ( + "NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6)" + ), + "tier_column": "NOW_GRPFTYP2", + "size_column": "NOEMP", + "sector_column": "PEIO1COW", + "payment_column": "NOW_HIPAID", + "payment_rule": ( + "1 (employer paid all): average total premium; 2 (some): average " + "total premium minus average employee contribution; 3 (none): 0" + ), + "suppressed_cell_fallback": ( + "national firm-size cell times the State total-to-national ratio" + ), + "government_cell_aging": ( + "MEPS-IC 2024 Series III x private-sector national 2025/2024 " + "ratio of the same tier and measure" + ), + "anchor": { + "source": EMPLOYER_PREMIUM_ANCHOR["source"], + "sha256": EMPLOYER_PREMIUM_ANCHOR["sha256"], + "series": EMPLOYER_PREMIUM_ANCHOR["series"], + "values": dict(EMPLOYER_PREMIUM_ANCHOR["values"]), + }, + } +) +_PRE_TAX_PARAMETERS: Mapping[str, Any] = MappingProxyType( + { + "seed_from_build_config": True, + "premium_column": "PHIP_VAL", + "wage_column": "WSAL_VAL", + "employer_payment_codes": [_PAYS_SOME, _PAYS_NONE], + "pre_tax_share": ( + "MEPS-IC 2025 Table I.A.2.j pretax-contribution offer percent / " + "Table I.A.2 health-insurance offer percent, by firm size" + ), + } +) + + +def us_esi_premiums_stage_spec() -> SourceStageSpec: + """Load the ``meps_esi_premiums`` stage and pin it to this contract.""" + + manifest = load_source_manifest( + files("microcosm.build.us").joinpath("source_stages.json") + ) + spec = manifest.stage_map().get(US_ESI_PREMIUMS_STAGE_NAME) + if spec is None: + raise ValueError( + f"US source manifest declares no {US_ESI_PREMIUMS_STAGE_NAME!r} stage." + ) + if spec.grain != "person": + raise ValueError("US ESI premium stage must have person grain.") + if tuple(spec.outputs) != US_ESI_PREMIUMS_OUTPUT_COLUMNS: + raise ValueError("US ESI premium manifest outputs drifted from the runtime.") + expected = ( + ("read_table", {"table": "person", "weight": _PERSON_WEIGHT_COLUMN}), + ( + "derive_employer_sponsored_insurance_premiums", + _plain(_EMPLOYER_PREMIUM_PARAMETERS), + ), + ("derive_pre_tax_health_insurance_premiums", _plain(_PRE_TAX_PARAMETERS)), + ) + actual = tuple( + (operation.kind, dict(operation.parameters)) for operation in spec.operations + ) + if actual != expected: + raise ValueError( + "US ESI premium operations drifted from the reviewed MEPS-IC cell " + "assignment and pre-tax share contract." + ) + return spec + + +def _plain(value: Any) -> Any: + if isinstance(value, Mapping): + return {key: _plain(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(item) for item in value] + return value + + +def load_meps_ic_esi_premium_cells() -> Mapping[str, Any]: + """The packaged MEPS-IC cell table, verified against its pinned digest.""" + + raw = ( + files("microcosm.build.us_runtime.data").joinpath(_CELLS_RESOURCE).read_bytes() + ) + digest = hashlib.sha256(raw).hexdigest() + if digest != _CELLS_SHA256: + raise SourceRuntimeError( + f"{_CELLS_RESOURCE} digest {digest} is not the reviewed {_CELLS_SHA256}; " + "regenerate it with tools/build_us_meps_ic_esi_cells.py and re-pin." + ) + return json.loads(raw) + + +@dataclass(frozen=True) +class _EsiCodes: + owner: np.ndarray + hipaid: np.ndarray + tier: np.ndarray + employed: np.ndarray + noemp: np.ndarray + sector: np.ndarray + state_fips: np.ndarray + + @property + def universe(self) -> np.ndarray: + return ( + (self.owner == _POLICYHOLDER) + & self.employed + & np.isin(self.sector, _EMPLOYER_SECTOR_CODES) + ) + + +def _integer_codes(person: pd.DataFrame, column: str) -> np.ndarray: + values = pd.to_numeric(person[column], errors="coerce") + numeric = values.to_numpy(dtype=np.float64, na_value=np.nan) + bad = ~np.isfinite(numeric) | (numeric != np.round(numeric)) + if bad.any(): + raise SourceRuntimeError( + f"US ESI premium stage: {column} has {int(bad.sum())} missing or " + "non-integer row(s); the Census column was not restored for every " + "vintage (asec_census_person_columns) and is never defaulted." + ) + codes = numeric.astype(np.int64) + domain = _CODE_DOMAINS.get(column) + if domain is not None: + outside = ~np.isin(codes, sorted(domain)) + if outside.any(): + raise SourceRuntimeError( + f"US ESI premium stage: {column} has {int(outside.sum())} code(s) " + f"outside the Census codebook {sorted(domain)}, e.g. " + f"{sorted(set(codes[outside].tolist()))[:5]}." + ) + return codes + + +def _esi_person_codes(person: pd.DataFrame) -> _EsiCodes: + """Validate the raw ASEC coverage, employment and employer fields. + + Reads NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR, NOEMP and + PEIO1COW, and refuses the Census universe relations the pinned person + members satisfy exactly: a policyholder reports a payment status and a + tier, everyone else reports neither, and NOW_GRPFTYP (family/self-only) + agrees with NOW_GRPFTYP2. + """ + + missing = [ + column + for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column not in person.columns + ] + if missing: + raise SourceRuntimeError( + "US ESI premium stage is source-unavailable: the person table lacks " + f"{missing}. Restore them from the pinned Census person files " + "(asec_census_person_columns); they are never proxied or defaulted." + ) + owner = _integer_codes(person, "NOW_OWNGRP") + hipaid = _integer_codes(person, "NOW_HIPAID") + grpftyp = _integer_codes(person, "NOW_GRPFTYP") + tier = _integer_codes(person, "NOW_GRPFTYP2") + pemlr = _integer_codes(person, "PEMLR") + noemp = _integer_codes(person, "NOEMP") + sector = _integer_codes(person, "PEIO1COW") + state_fips = _integer_codes(person, "state_fips") + holder = owner == _POLICYHOLDER + violations = { + "policyholder_without_payment_status": int( + (holder & ~np.isin(hipaid, [_PAYS_ALL, _PAYS_SOME, _PAYS_NONE])).sum() + ), + "policyholder_without_tier": int((holder & ~np.isin(tier, [1, 2, 3])).sum()), + "non_policyholder_with_payment_status": int((~holder & (hipaid != 0)).sum()), + "non_policyholder_with_tier": int((~holder & (tier != 0)).sum()), + "grpftyp_disagrees_with_grpftyp2": int( + ( + holder + & ( + grpftyp + != pd.Series(tier) + .map(dict(_GRPFTYP_FOR_GRPFTYP2)) + .fillna(-1) + .to_numpy() + ) + ).sum() + ), + } + if any(violations.values()): + rendered = ", ".join( + f"{key}={value}" for key, value in violations.items() if value + ) + raise SourceRuntimeError( + f"US ESI premium stage refused the ASEC coverage fields: {rendered}." + ) + return _EsiCodes( + owner=owner, + hipaid=hipaid, + tier=tier, + employed=np.isin(pemlr, _EMPLOYED), + noemp=noemp, + sector=sector, + state_fips=state_fips, + ) + + +def _private_cell( + cells: Mapping[str, Any], tier: str, measure: str, state: str, size: str +) -> float: + rows = cells["private_state_2025"][tier][measure]["rows"] + if state not in rows: + raise SourceRuntimeError( + f"US ESI premium stage: no MEPS-IC row for State {state}." + ) + value = rows[state][size] + if value is None: + # AHRQ suppressed the cell: national firm-size cell scaled by the + # State's all-sizes level relative to the nation. + national = rows["US"] + value = national[size] * rows[state]["total"] / national["total"] + return float(value) + + +def _government_cell( + cells: Mapping[str, Any], tier: str, measure: str, division: str, column: str +) -> float: + value = cells["public_division_2024"][tier][measure]["rows"][division][column] + current = cells["private_state_2025"][tier][measure]["rows"]["US"]["total"] + prior = cells["private_national_2024"][tier][measure]["total"] + return float(value) * float(current) / float(prior) + + +def _cell_values( + codes: _EsiCodes, cells: Mapping[str, Any] +) -> tuple[np.ndarray, np.ndarray]: + """Average total premium and employee contribution of each person's cell. + + NaN outside the employed-policyholder universe. + """ + + premium = np.full(len(codes.owner), np.nan) + contribution = np.full(len(codes.owner), np.nan) + divisions = cells["state_census_division"] + keys = pd.DataFrame( + { + "tier": codes.tier, + "noemp": codes.noemp, + "sector": codes.sector, + "state": codes.state_fips, + } + ) + universe = codes.universe + for (tier_code, noemp, sector, state), index in ( + keys[universe].groupby(["tier", "noemp", "sector", "state"]).groups.items() + ): + tier = _TIER_BY_CODE[int(tier_code)] + state_key = f"{int(state):02d}" + if state_key not in divisions: + raise SourceRuntimeError( + f"US ESI premium stage: State FIPS {int(state)} has no MEPS-IC cell." + ) + if int(sector) in _PRIVATE_EMPLOYER_CODES: + size = _SIZE_BY_NOEMP[int(noemp)] + values = [ + _private_cell(cells, tier, measure, state_key, size) + for measure in ("premium", "employee_contribution") + ] + else: + column = ( + "state" if int(sector) == _STATE_GOVERNMENT else "all_state_and_local" + ) + values = [ + _government_cell(cells, tier, measure, divisions[state_key], column) + for measure in ("premium", "employee_contribution") + ] + rows = np.asarray(index, dtype=np.int64) + premium[rows], contribution[rows] = values + return premium, contribution + + +def _raw_employer_share( + codes: _EsiCodes, premium: np.ndarray, contribution: np.ndarray +) -> np.ndarray: + raw = np.zeros(len(codes.owner), dtype=np.float64) + universe = codes.universe + pays_all = universe & (codes.hipaid == _PAYS_ALL) + pays_some = universe & (codes.hipaid == _PAYS_SOME) + raw[pays_all] = premium[pays_all] + raw[pays_some] = np.clip(premium[pays_some] - contribution[pays_some], 0.0, None) + return raw + + +def _anchor(year: int) -> float: + values = EMPLOYER_PREMIUM_ANCHOR["values"] + if str(year) not in values: + raise SourceRuntimeError( + f"US ESI premium stage has no BEA anchor for {year}; pinned years: " + f"{sorted(values)}." + ) + return float(values[str(year)]) + + +def _check_parameters( + operation: SourceOperationSpec, expected: Mapping[str, Any] +) -> None: + if dict(operation.parameters) != _plain(expected): + raise SourceRuntimeError( + f"US ESI premium operation {operation.kind!r} parameters drifted from " + "the reviewed contract." + ) + + +def _person_weights(frame: pd.DataFrame) -> np.ndarray: + if _PERSON_WEIGHT_COLUMN not in frame.columns: + raise SourceRuntimeError("US ESI premium stage requires person weights.") + weights = frame[_PERSON_WEIGHT_COLUMN].to_numpy(dtype=np.float64) + if not np.isfinite(weights).all() or (weights < 0).any(): + raise SourceRuntimeError( + "US ESI premium stage requires finite nonnegative weights." + ) + return weights + + +def derive_us_employer_esi_premiums_from_manifest( + frame: pd.DataFrame | None, + operation: SourceOperationSpec, + context: SourceRuntimeContext, +) -> pd.DataFrame: + """Assign MEPS-IC employer shares and scale them to the BEA anchor.""" + + if operation.kind != "derive_employer_sponsored_insurance_premiums": + raise SourceRuntimeError( + f"Unexpected ESI premium operation {operation.kind!r}." + ) + if frame is None: + raise SourceRuntimeError("ESI premium assignment requires the person table.") + _check_parameters(operation, _EMPLOYER_PREMIUM_PARAMETERS) + codes = _esi_person_codes(frame) + premium, contribution = _cell_values(codes, load_meps_ic_esi_premium_cells()) + raw = _raw_employer_share(codes, premium, contribution) + weights = _person_weights(frame) + raw_total = float(weights @ raw) + if not raw_total > 0: + raise SourceRuntimeError( + "US ESI premium stage found no weighted employer-paid premium mass to " + "scale: no employed policyholder with an employer share." + ) + scale = _anchor(context.config.target_year) / raw_total + result = frame.copy(deep=True) + result[US_ESI_EMPLOYER_PREMIUM_COLUMN] = raw * scale + return result + + +def _pre_tax_shares(cells: Mapping[str, Any]) -> dict[str, float]: + rows = cells["pretax_contribution_2025"]["rows"] + return { + size: rows[size]["pretax_contribution_offer_percent"] + / rows[size]["health_insurance_offer_percent"] + for size in ("total", "lt50", "50plus") + } + + +def _stable_person_keys(frame: pd.DataFrame) -> pd.Series: + if {"source_year", "source_household_id", "source_person_id"} <= set(frame.columns): + return ( + frame["source_year"].astype(str) + + ":" + + frame["source_household_id"].astype(str) + + ":" + + frame["source_person_id"].astype(str) + ) + return frame["person_id"].astype(str) + + +def _stable_person_draws(frame: pd.DataFrame, *, seed: int) -> np.ndarray: + """Seeded uniform draws keyed by stable source identity per person. + + Support-channel clones share their source identity, so they always + receive the same draw; frames without source columns key on the person + id itself. + """ + + denominator = float(2**64) + return np.asarray( + [ + int.from_bytes( + hashlib.blake2b( + f"{seed}:{_DRAW_SALT}:{key}".encode(), + digest_size=8, + ).digest(), + byteorder="big", + signed=False, + ) + / denominator + for key in _stable_person_keys(frame) + ], + dtype=np.float64, + ) + + +def _pre_tax_eligible(frame: pd.DataFrame, codes: _EsiCodes) -> np.ndarray: + premium = pd.to_numeric(frame["PHIP_VAL"], errors="coerce").to_numpy( + dtype=np.float64 + ) + wages = pd.to_numeric(frame["WSAL_VAL"], errors="coerce").to_numpy(dtype=np.float64) + if not (np.isfinite(premium).all() and np.isfinite(wages).all()): + raise SourceRuntimeError( + "US ESI premium stage requires finite PHIP_VAL and WSAL_VAL." + ) + return ( + codes.universe + & np.isin(codes.hipaid, [_PAYS_SOME, _PAYS_NONE]) + & (wages > 0) + & (premium > 0) + ) + + +def derive_us_pre_tax_health_insurance_premiums_from_manifest( + frame: pd.DataFrame | None, + operation: SourceOperationSpec, + context: SourceRuntimeContext, +) -> pd.DataFrame: + """Route an eligible policyholder's reported premium through payroll.""" + + if operation.kind != "derive_pre_tax_health_insurance_premiums": + raise SourceRuntimeError( + f"Unexpected ESI premium operation {operation.kind!r}." + ) + if frame is None: + raise SourceRuntimeError( + "Pre-tax premium assignment requires the person table." + ) + _check_parameters(operation, _PRE_TAX_PARAMETERS) + codes = _esi_person_codes(frame) + eligible = _pre_tax_eligible(frame, codes) + shares = _pre_tax_shares(load_meps_ic_esi_premium_cells()) + probability = ( + pd.Series(codes.noemp).map(dict(_SIZE_BY_NOEMP)).map(shares).to_numpy() + ) + draws = _stable_person_draws(frame, seed=context.config.seed) + premium = pd.to_numeric(frame["PHIP_VAL"]).to_numpy(dtype=np.float64) + result = frame.copy(deep=True) + result[US_ESI_PRE_TAX_PREMIUM_COLUMN] = np.where( + eligible & (draws < probability), premium, 0.0 + ) + return result + + +def _has_raw_columns(person: pd.DataFrame) -> bool: + return set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) <= set(person.columns) + + +def _person_with_state(frame: Frame) -> pd.DataFrame: + """The person table with the household ``state_fips`` joined on.""" + + person = frame.table("person").copy(deep=True) + if "state_fips" in person.columns: + return person + household = frame.table("household") + if "state_fips" not in household.columns: + return person + state = household.set_index("household_id")["state_fips"] + if not state.index.is_unique: + raise SourceRuntimeError("US ESI premium stage: household ids repeat.") + joined = person["person_household_id"].map(state) + if joined.isna().any(): + raise SourceRuntimeError( + f"US ESI premium stage: {int(joined.isna().sum())} person(s) have no " + "household state_fips." + ) + person["state_fips"] = joined.to_numpy() + return person + + +def _outputs_carry_signal(person: pd.DataFrame) -> bool: + return all( + column in person.columns and person[column].dropna().nunique() > 1 + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + ) + + +def with_us_esi_premium_inputs(frame: Frame, *, seed: int, time_period: int) -> Frame: + """Run the ``meps_esi_premiums`` stage over a US frame. + + A frame that already carries both outputs with signal and passes the + signal gate is returned unchanged (idempotent). Otherwise the person table + must still carry the raw ASEC columns; the stage never defaults them. + """ + + if frame.schema != US_SCHEMA: + raise ValueError("US ESI premium inputs require the US schema.") + person = frame.table("person") + if _outputs_carry_signal(person): + gate = us_esi_premiums_signal_gate(frame) + if not gate.passed: + raise SourceRuntimeError( + "US ESI premium preexisting inputs fail the signal gate: " + + "; ".join(gate.failures) + ) + return frame + stage_person = _person_with_state(frame) + stage_person[_PERSON_WEIGHT_COLUMN] = frame.resolve_weights("person").values + output = run_source_stage( + us_esi_premiums_stage_spec(), + tables={"person": stage_person}, + operation_handlers={ + "derive_employer_sponsored_insurance_premiums": ( + derive_us_employer_esi_premiums_from_manifest + ), + "derive_pre_tax_health_insurance_premiums": ( + derive_us_pre_tax_health_insurance_premiums_from_manifest + ), + }, + config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), + ) + aligned = output.set_index("person_id").reindex(person["person_id"]) + tables = {entity: frame.table(entity).copy() for entity in frame.entities} + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + values = aligned[column].to_numpy(dtype=np.float64) + if not np.isfinite(values).all(): + raise ValueError(f"US ESI premium stage output does not cover {column!r}.") + tables["person"][column] = values + return Frame( + tables, + frame.schema, + {entity: frame.weights_for(entity) for entity in frame.weighted_entities}, + frame.strata, + mass_log=frame.mass_log, + metadata=frame.metadata, + ) + + +def _weighted( + weights: np.ndarray, values: np.ndarray, mask: np.ndarray | None = None +) -> float: + if mask is None: + return float(weights @ values) + return float(weights[mask] @ values[mask]) + + +def _clone_disagreements(person: pd.DataFrame) -> int: + if not {"source_year", "source_household_id", "source_person_id"} <= set( + person.columns + ): + return 0 + keys = _stable_person_keys(person) + work = person[list(US_ESI_PREMIUMS_OUTPUT_COLUMNS)].copy() + work["_key"] = keys.to_numpy() + return int( + work.groupby("_key", sort=False)[list(US_ESI_PREMIUMS_OUTPUT_COLUMNS)] + .nunique() + .gt(1) + .any(axis=1) + .sum() + ) + + +def us_esi_premiums_summary(frame: Frame) -> dict[str, object]: + """Weighted ESI premium lineage and structure, for gates and manifests. + + With the raw ASEC columns present it also recomputes the raw MEPS-IC + shares, so the scale factor and every structural zero are checked against + the cells rather than trusted. + """ + + person = frame.table("person") + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + pre_tax = person[US_ESI_PRE_TAX_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + positive = employer > 0 + total_weight = float(weights.sum()) + holders = float(weights[positive].sum()) + summary: dict[str, object] = { + "employer_premium_total": _weighted(weights, employer), + "employer_premium_positive_persons": holders, + "employer_premium_mean_per_positive_person": ( + _weighted(weights, employer, positive) / holders if holders else 0.0 + ), + "employer_premium_positive_share": holders / total_weight + if total_weight + else 0.0, + "pre_tax_premium_total": _weighted(weights, pre_tax), + "pre_tax_premium_positive_persons": float(weights[pre_tax > 0].sum()), + "pre_tax_premium_positive_share": ( + float(weights[pre_tax > 0].sum()) / total_weight if total_weight else 0.0 + ), + "nonfinite_rows": int( + (~np.isfinite(employer)).sum() + (~np.isfinite(pre_tax)).sum() + ), + "negative_rows": int((employer < 0).sum() + (pre_tax < 0).sum()), + "clone_disagreement_source_persons": _clone_disagreements(person), + "anchor": dict(EMPLOYER_PREMIUM_ANCHOR), + "cross_check": dict(EMPLOYER_PREMIUM_CROSS_CHECK), + "cells_sha256": _CELLS_SHA256, + } + person = _person_with_state(frame) + if not _has_raw_columns(person): + return summary + codes = _esi_person_codes(person) + cells = load_meps_ic_esi_premium_cells() + premium, contribution = _cell_values(codes, cells) + raw = _raw_employer_share(codes, premium, contribution) + raw_positive = raw > 0 + raw_total = _weighted(weights, raw) + scale = summary["employer_premium_total"] / raw_total if raw_total else float("nan") + eligible = _pre_tax_eligible(person, codes) + phip = pd.to_numeric(person["PHIP_VAL"]).to_numpy(dtype=np.float64) + holder = codes.owner == _POLICYHOLDER + by_tier = { + tier: _weighted(weights, employer, codes.universe & (codes.tier == code)) + for code, tier in _TIER_BY_CODE.items() + } + by_sector = { + "private": _weighted( + weights, employer, np.isin(codes.sector, _PRIVATE_EMPLOYER_CODES) + ), + "federal": _weighted(weights, employer, codes.sector == _FEDERAL), + "state_and_local": _weighted( + weights, + employer, + np.isin(codes.sector, [_STATE_GOVERNMENT, _LOCAL_GOVERNMENT]), + ), + } + with np.errstate(invalid="ignore", divide="ignore"): + ratio = np.where( + raw_positive, employer / np.where(raw_positive, raw, 1.0), np.nan + ) + summary |= { + "weighted_policyholders": float(weights[holder].sum()), + "weighted_employed_policyholders": float(weights[codes.universe].sum()), + "weighted_employed_policyholders_by_payment": { + name: float(weights[codes.universe & (codes.hipaid == code)].sum()) + for name, code in ( + ("all", _PAYS_ALL), + ("some", _PAYS_SOME), + ("none", _PAYS_NONE), + ) + }, + "raw_employer_share_total": raw_total, + "raw_employer_share_mean_per_positive_person": ( + _weighted(weights, raw, raw_positive) / float(weights[raw_positive].sum()) + if raw_positive.any() + else 0.0 + ), + "raw_employer_share_band": list(_RAW_MEAN_BAND), + "scale_factor": scale, + "scale_factor_spread": ( + float(np.nanmax(ratio) - np.nanmin(ratio)) if raw_positive.any() else 0.0 + ), + "employer_premium_by_tier": by_tier, + "employer_premium_by_sector": by_sector, + "employer_premium_outside_universe_rows": int( + (positive & ~codes.universe).sum() + ), + "employer_premium_where_employer_pays_none_rows": int( + (positive & (codes.hipaid == _PAYS_NONE)).sum() + ), + "employer_premium_without_raw_share_rows": int( + (positive & ~raw_positive).sum() + ), + "pre_tax_ineligible_rows": int(((pre_tax > 0) & ~eligible).sum()), + "pre_tax_not_reported_premium_rows": int( + ((pre_tax > 0) & ~np.isclose(pre_tax, phip)).sum() + ), + "pre_tax_eligible_weight": float(weights[eligible].sum()), + "pre_tax_selected_share_of_eligible": ( + float(weights[eligible & (pre_tax > 0)].sum()) + / float(weights[eligible].sum()) + if eligible.any() + else 0.0 + ), + "pre_tax_shares": _pre_tax_shares(cells), + } + return summary + + +def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: + """Structure and scale-invariant signal of the two ESI premium inputs. + + Holds on any frame size (smoke builds included). On frames that still + carry the raw ASEC columns it also proves every positive value sits in the + employed-policyholder universe, equals one common multiple of its MEPS-IC + cell, and that pre-tax premiums are reported premiums of eligible people. + """ + + person = frame.table("person") + missing = [ + column + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="esi_premiums_signal", + passed=False, + failures=tuple(f"person column missing: {column}." for column in missing), + details={"missing": missing}, + ) + try: + summary = us_esi_premiums_summary(frame) + except SourceRuntimeError as exc: + return GateResult( + name="esi_premiums_signal", + passed=False, + failures=(str(exc),), + details={"structural_error": str(exc)}, + ) + failures: list[str] = [] + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + if person[column].dropna().nunique() < 2: + failures.append(f"{column}: constant column — zero mass or no signal.") + for key, label in ( + ("nonfinite_rows", "non-finite value(s)"), + ("negative_rows", "negative value(s)"), + ("clone_disagreement_source_persons", "support-clone disagreement(s)"), + ( + "employer_premium_outside_universe_rows", + "employer premium(s) outside employed policyholders", + ), + ( + "employer_premium_where_employer_pays_none_rows", + "employer premium(s) where the employer pays none", + ), + ( + "employer_premium_without_raw_share_rows", + "employer premium(s) with no MEPS-IC share", + ), + ("pre_tax_ineligible_rows", "pre-tax premium(s) for ineligible people"), + ( + "pre_tax_not_reported_premium_rows", + "pre-tax premium(s) not equal to PHIP_VAL", + ), + ): + count = int(summary.get(key, 0)) + if count: + failures.append(f"ESI premiums: {count} {label}.") + share = float(summary["employer_premium_positive_share"]) + low, high = _POSITIVE_SHARE_BAND + if not low <= share <= high: + failures.append( + f"employer premium positive share {share:.4f} outside [{low}, {high}]." + ) + share = float(summary["pre_tax_premium_positive_share"]) + low, high = _PRE_TAX_POSITIVE_SHARE_BAND + if not low <= share <= high: + failures.append( + f"pre-tax premium positive share {share:.4f} outside [{low}, {high}]." + ) + if "raw_employer_share_mean_per_positive_person" in summary: + mean = float(summary["raw_employer_share_mean_per_positive_person"]) + low, high = _RAW_MEAN_BAND + if not low <= mean <= high: + failures.append( + f"raw MEPS-IC employer share per holder ${mean:,.0f} outside " + f"[${low:,.0f}, ${high:,.0f}]: the cell table is broken." + ) + # Relative to the factor: a float32 round trip leaves ~1e-7. + spread = float(summary["scale_factor_spread"]) + if spread > 1e-6 * float(summary["scale_factor"]): + failures.append( + f"employer premiums are not one common multiple of their MEPS-IC " + f"cells (scale-factor spread {spread:.3g})." + ) + return GateResult( + name="esi_premiums_signal", + passed=not failures, + failures=tuple(failures), + details=summary, + ) + + +def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult: + """Hold a release's weighted ESI premium totals to their external anchors. + + Runs on the calibrated export (dense and sparse). Fails when either column + is absent or zero-mass, when the employer premium total leaves + ±``ANCHOR_RELATIVE_TOLERANCE`` of BEA NIPA 7.8 line 17 for ``time_period`` + or exceeds the retiree-inclusive NHE Table 24 employer contribution, or + when pre-tax premiums exceed NHE's employee contribution (which also + counts retiree- and COBRA-paid premiums). + """ + + person = frame.table("person") + missing = [ + column + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + if column not in person.columns + ] + if missing: + return GateResult( + name="esi_premiums_anchor", + passed=False, + failures=tuple(f"person column missing: {column}." for column in missing), + details={"missing": missing}, + ) + weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) + employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + pre_tax = person[US_ESI_PRE_TAX_PREMIUM_COLUMN].to_numpy(dtype=np.float64) + employer_total = _weighted(weights, employer) + pre_tax_total = _weighted(weights, pre_tax) + year = str(int(time_period)) + failures: list[str] = [] + details: dict[str, object] = { + "time_period": int(time_period), + "employer_premium_total": employer_total, + "pre_tax_premium_total": pre_tax_total, + "employer_premium_positive_persons": float(weights[employer > 0].sum()), + "relative_tolerance": ANCHOR_RELATIVE_TOLERANCE, + "anchor": dict(EMPLOYER_PREMIUM_ANCHOR), + "cross_check": dict(EMPLOYER_PREMIUM_CROSS_CHECK), + } + for column, total in ( + (US_ESI_EMPLOYER_PREMIUM_COLUMN, employer_total), + (US_ESI_PRE_TAX_PREMIUM_COLUMN, pre_tax_total), + ): + if not total > 0: + failures.append(f"{column}: zero weighted mass.") + anchor = EMPLOYER_PREMIUM_ANCHOR["values"].get(year) + if anchor is None: + failures.append(f"no BEA NIPA 7.8 line 17 anchor for {year}.") + else: + relative = employer_total / float(anchor) - 1.0 + details |= {"anchor_value": float(anchor), "relative_error": relative} + if abs(relative) > ANCHOR_RELATIVE_TOLERANCE: + failures.append( + f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: weighted total " + f"${employer_total / 1e9:,.1f}B is {relative:+.1%} from BEA NIPA " + f"7.8 line 17 ${float(anchor) / 1e9:,.1f}B ({year}); tolerance " + f"±{ANCHOR_RELATIVE_TOLERANCE:.0%}." + ) + nhe_employer = EMPLOYER_PREMIUM_CROSS_CHECK["employer_contribution"].get(year) + nhe_employee = EMPLOYER_PREMIUM_CROSS_CHECK["employee_contribution"].get(year) + if nhe_employer is not None: + details["cross_check_employer_ratio"] = employer_total / float(nhe_employer) + if employer_total > float(nhe_employer): + failures.append( + f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: weighted total " + f"${employer_total / 1e9:,.1f}B exceeds the retiree-inclusive NHE " + f"Table 24 employer contribution ${float(nhe_employer) / 1e9:,.1f}B." + ) + if nhe_employee is not None: + details["cross_check_pre_tax_ratio"] = pre_tax_total / float(nhe_employee) + if pre_tax_total > float(nhe_employee): + failures.append( + f"{US_ESI_PRE_TAX_PREMIUM_COLUMN}: weighted total " + f"${pre_tax_total / 1e9:,.1f}B exceeds NHE Table 24's employee " + f"contribution ${float(nhe_employee) / 1e9:,.1f}B." + ) + return GateResult( + name="esi_premiums_anchor", + passed=not failures, + failures=tuple(failures), + details=details, + ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index e803bb0137..40856b549c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -70,6 +70,10 @@ US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, ) from microcosm.build.us_runtime.energy_subsidy import US_ENERGY_SUBSIDY_OUTPUT_COLUMNS +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PRE_TAX_PREMIUM_COLUMN, +) from microcosm.build.us_runtime.farm_business_income import ( US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, ) @@ -189,6 +193,10 @@ # pinned Census ASEC person files; the certified default carries it # through the Build P source enrichment. *US_SPM_INDEPENDENCE_ROLE_OUTPUT_COLUMNS, + # PolicyEngine/microcosm#454: the employee section 125 premium leaf the + # meps_esi_premiums base-builder stage writes beside the restored + # employer premium. Absent, income-tax and FICA wages both overstate. + US_ESI_PRE_TAX_PREMIUM_COLUMN, } ) @@ -209,6 +217,7 @@ *US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, *US_DOMESTIC_PRODUCTION_ALD_OUTPUT_COLUMNS, *US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, + US_ESI_EMPLOYER_PREMIUM_COLUMN, *US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, *US_FORM_4952_OUTPUT_COLUMNS, *US_CAPITAL_GAIN_DETAILS_OUTPUT_COLUMNS, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py index a67869a0e8..6bff894ce9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py @@ -42,6 +42,10 @@ derive_us_energy_subsidy_from_manifest, impute_us_energy_subsidy_to_puf_support_from_manifest, ) +from microcosm.build.us_runtime.esi_premiums import ( + derive_us_employer_esi_premiums_from_manifest, + derive_us_pre_tax_health_insurance_premiums_from_manifest, +) from microcosm.build.us_runtime.hours_worked import ( derive_us_hours_worked_from_manifest, ) @@ -315,10 +319,16 @@ def us_source_operation_handlers() -> Mapping[str, SourceOperationHandler]: derive_us_other_health_insurance_from_manifest ), "derive_eligibility_inputs": derive_us_eligibility_inputs_from_manifest, + "derive_employer_sponsored_insurance_premiums": ( + derive_us_employer_esi_premiums_from_manifest + ), "derive_education_inputs": derive_us_education_inputs_from_manifest, "derive_hours_worked": derive_us_hours_worked_from_manifest, "derive_medicare_take_up": derive_us_medicare_take_up_from_manifest, "derive_pregnancy": derive_us_pregnancy_from_manifest, + "derive_pre_tax_health_insurance_premiums": ( + derive_us_pre_tax_health_insurance_premiums_from_manifest + ), "derive_prior_year_income": derive_us_prior_year_income_from_manifest, "derive_relationship_inputs": derive_us_relationship_inputs_from_manifest, "derive_spm_independence_role": (derive_us_spm_independence_role_from_manifest), diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 7be8062b02..571bbc9d0d 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_184 - assert fields["authored_normative_field_count"] == 32_404 - assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_184 + assert fields["configuration_field_count"] == 42_252 + assert fields["authored_normative_field_count"] == 32_452 + assert fields["resolved_binding_field_count"] == 9_800 + assert fields["consumed_field_count"] == 42_252 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_055, + "compiler_semantic": 27_740, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_543, + "no_generation0_effect": 3_709, } inventory = coverage_report["inventory_coverage"] @@ -108,7 +108,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["counts"]["producer_inputs"] == 2_744 assert inventory["counts"]["ownership_rows"] == 18 assert inventory["counts"]["tail_control_fields"] == 934 - assert inventory["counts"]["seed_owner_bindings"] == 112 + assert inventory["counts"]["seed_owner_bindings"] == 113 @pytest.mark.parametrize( diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py b/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py new file mode 100644 index 0000000000..a4647c46cb --- /dev/null +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py @@ -0,0 +1,114 @@ +"""Engine contract of the two ESI premium inputs (microcosm #454). + +``docs/us-esi-employer-premiums.md`` and the ``meps_esi_premiums`` stage notes +say how PolicyEngine-US consumes each input. These tests compute it in the +locked engine, so an engine release that reroutes either input fails here +rather than silently changing what the stage's columns mean. +""" + +from __future__ import annotations + +import pytest + +from microcosm.build.us_runtime import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PRE_TAX_PREMIUM_COLUMN, +) + +PERIOD = 2024 +WAGES = 60_000.0 +MEASURES = ( + "irs_employment_income", + "payroll_tax_gross_wages", + "adjusted_gross_income", + "income_tax", + "employee_social_security_tax", + "cbo_household_market_income", +) + + +def _totals(**person_inputs: float) -> dict[str, float]: + from policyengine_us import Simulation + + person = {"age": {PERIOD: 40}, "employment_income": {PERIOD: WAGES}} + person |= {name: {PERIOD: value} for name, value in person_inputs.items()} + simulation = Simulation( + situation={ + "people": {"worker": person}, + "tax_units": {"tax_unit": {"members": ["worker"]}}, + "spm_units": {"spm_unit": {"members": ["worker"]}}, + "families": {"family": {"members": ["worker"]}}, + "marital_units": {"marital_unit": {"members": ["worker"]}}, + "households": { + "household": {"members": ["worker"], "state_code": {PERIOD: "TX"}} + }, + } + ) + return { + measure: float(simulation.calculate(measure, PERIOD).sum()) + for measure in MEASURES + } + + +@pytest.fixture(scope="module") +def baseline() -> dict[str, float]: + return _totals() + + +def test_both_inputs_are_person_year_input_leaves() -> None: + from policyengine_us import CountryTaxBenefitSystem + + variables = CountryTaxBenefitSystem().variables + for name in (US_ESI_EMPLOYER_PREMIUM_COLUMN, US_ESI_PRE_TAX_PREMIUM_COLUMN): + variable = variables[name] + assert variable.is_input_variable(), name + assert variable.entity.key == "person", name + assert str(variable.definition_period).lower() == "year", name + + +def test_the_engine_parameter_lists_name_the_inputs() -> None: + from policyengine_us import CountryTaxBenefitSystem + + parameters = CountryTaxBenefitSystem().parameters + instant = f"{PERIOD}-01-01" + assert US_ESI_EMPLOYER_PREMIUM_COLUMN in ( + parameters.gov.household.cbo_market_income_additions(instant) + ) + assert US_ESI_PRE_TAX_PREMIUM_COLUMN in ( + parameters.gov.irs.gross_income.pre_tax_contributions(instant) + ) + assert US_ESI_PRE_TAX_PREMIUM_COLUMN in ( + parameters.gov.irs.gross_income.fica_pre_tax_contributions(instant) + ) + + +def test_a_pre_tax_premium_lowers_income_tax_and_fica_wages_dollar_for_dollar( + baseline, +) -> None: + premium = 2_000.0 + reduced = _totals(**{US_ESI_PRE_TAX_PREMIUM_COLUMN: premium}) + + for measure in ( + "irs_employment_income", + "payroll_tax_gross_wages", + "adjusted_gross_income", + ): + assert reduced[measure] - baseline[measure] == pytest.approx(-premium), measure + assert baseline["irs_employment_income"] == pytest.approx(WAGES) + assert reduced["income_tax"] < baseline["income_tax"] + # 6.2% employee OASDI on the premium no longer in FICA wages. + assert reduced["employee_social_security_tax"] - baseline[ + "employee_social_security_tax" + ] == pytest.approx(-0.062 * premium) + + +def test_an_employer_premium_moves_only_cbo_household_market_income(baseline) -> None: + premium = 10_000.0 + with_premium = _totals(**{US_ESI_EMPLOYER_PREMIUM_COLUMN: premium}) + + assert with_premium["cbo_household_market_income"] - baseline[ + "cbo_household_market_income" + ] == pytest.approx(premium) + for measure in MEASURES: + if measure != "cbo_household_market_income": + assert with_premium[measure] == pytest.approx(baseline[measure]), measure diff --git a/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py b/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py index 3717bc36bc..43eea6d2c3 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_release_target_parity.py @@ -315,7 +315,7 @@ def test_manifest_header_counts_match_parsed_families(self) -> None: manifest.reviewed_exclusions ) assert len(manifest.compiled_families) == 32 - assert len(manifest.reviewed_exclusions) == 52 + assert len(manifest.reviewed_exclusions) == 53 def test_wired_nipa_and_liheap_families_are_compiled(self) -> None: manifest = load_target_parity_manifest() diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py index b025205ae5..e0ed3f5e12 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py @@ -107,9 +107,9 @@ def test_us_seed_stream_map_is_complete_and_owner_typed( ) -> None: seed_map = compiled_us.seed_stream_map assert seed_map.protocol_id == "legacy-v1" - assert len(seed_map.sites) == 53 - assert len(seed_map.owners) == 54 - assert sum(len(site.owners) for site in seed_map.sites) == 112 + assert len(seed_map.sites) == 54 + assert len(seed_map.owners) == 55 + assert sum(len(site.owners) for site in seed_map.sites) == 113 assert {site.id for site in seed_map.sites} == { site.id for site in resolved_us.seed_protocol.sites } @@ -244,5 +244,5 @@ def test_bundle_without_imputation_compiles_empty_graph( assert compiled.producer_graph.nodes == () assert compiled.stage_dag.nodes == () assert compiled.nodes == () - assert len(compiled.seed_stream_map.sites) == 53 + assert len(compiled.seed_stream_map.sites) == 54 assert compiled.seed_stream_map.owners == () diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index c5ec18bb51..2e7a1cd068 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_184 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_252 assert field_ledger.source_counts == { - "authored": 32_404, - "resolved_bindings": 9_780, + "authored": 32_452, + "resolved_bindings": 9_800, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_010, - "compiler_semantic": 27_717, + "legacy_behavior": 14_055, + "compiler_semantic": 27_740, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_498, - "no_generation0_effect": 3_686, + "legacy_behavior": 38_543, + "no_generation0_effect": 3_709, } - assert len({field.pointer for field in field_ledger.fields}) == 42_184 + assert len({field.pointer for field in field_ledger.fields}) == 42_252 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py index fce631f3a5..73ce670319 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py @@ -118,13 +118,13 @@ def test_identity_contract_objects_are_closed_world( load_schema_registry().validate(mutated, "spine.schema.json") -def test_all_53_seed_sites_resolve_to_typed_real_owners( +def test_all_54_seed_sites_resolve_to_typed_real_owners( identity_documents: tuple[dict[str, object], list[dict[str, object]]], ) -> None: spine, source_stages = identity_documents bindings = _resolve(spine, source_stages) - assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 54 assert [binding.site for binding in bindings] == [ site.id for site in LEGACY_V1_PROTOCOL.sites ] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 17cd0b715a..7be042baf6 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "69de1ffc232c3a4da09925658c9d105af55bdaae90852824ac6283306932aab9" + "a48d950ffc0079ee67a4994e0b8df8d25a9ad4d660dc7e9ef0d3337df55e49d9" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py index 78df801e0d..8125365db7 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py @@ -42,7 +42,7 @@ def test_plan_lock_is_complete_and_closed_world(compiled_us: CompiledSpecIR) -> assert payload["spec_binding"]["attestation"] == "mirror-attested" assert len(payload["producer_graph"]["nodes"]) == 38 assert len(payload["producer_graph"]["authored"]["ownership_matrix"]) == 18 - assert len(payload["seed_stream_map"]["sites"]) == 53 + assert len(payload["seed_stream_map"]["sites"]) == 54 assert len(payload["nodes"]) == 38 mutated = copy.deepcopy(payload) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py index bba3a43b43..7b1e53311e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py @@ -50,6 +50,7 @@ "legacy_puma_ladder", "medicaid_take_up_assignment", "other_health_insurance_training_cap", + "pre_tax_health_premium_assignment", "pregnancy_assignment", "primary_qrf_fit_draw", "prior_year_income_training_cap", @@ -136,6 +137,7 @@ ), "snap_take_up_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/snap_take_up.py", "pregnancy_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/pregnancy.py", + "pre_tax_health_premium_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py", "wic_claim_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/wic_claim.py", "snap_discretionary_exemption_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/snap_discretionary_exemption.py", **dict.fromkeys( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py b/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py index ecb0362987..2fad301645 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_asec_census_person_columns.py @@ -68,6 +68,12 @@ } ) +#: The six columns microcosm #454 restores for the ``meps_esi_premiums`` stage. +#: No pinned H5 carries them, and they were never part of the #720 offline fix. +_ISSUE_454_ADDED = frozenset( + {"NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2", "PEMLR", "NOEMP"} +) + #: H5 row order differs from the member's, so a positional copy would fail. _H5_ORDER = [3, 0, 7, 5, 1, 6, 2, 4] @@ -367,7 +373,9 @@ def test_the_review_places_every_offline_fix_column_exactly_once(): not_restored = set(ASEC_CENSUS_PERSON_COLUMNS_NOT_RESTORED) assert len(restored) == len(ASEC_CENSUS_PERSON_COLUMN_NAMES) assert restored.isdisjoint(not_restored) - assert restored | not_restored == _RECEIPT_720_ADDED + assert restored | not_restored == _RECEIPT_720_ADDED | _ISSUE_454_ADDED + assert _ISSUE_454_ADDED <= restored + assert _ISSUE_454_ADDED.isdisjoint(_RECEIPT_720_ADDED) assert not restored & set(ASEC_CENSUS_PERSON_IDENTITY_COLUMNS) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index a780c4814b..c5203a6203 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf", + "sha256": "0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py index 5b90126482..366e62eb3f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py @@ -128,7 +128,7 @@ def test_parity_resources_and_generator_restate_the_pin() -> None: assert manifest["reference"]["feed"] == generator.DEFAULT_FEED_NAME assert generator.DEFAULT_FEED_PATH.name == generator.DEFAULT_FEED_NAME assert manifest["reference"]["compiled_families"] == "32" - assert manifest["reference"]["reviewed_exclusions"] == "52" + assert manifest["reference"]["reviewed_exclusions"] == "53" def test_scope_names_every_pinned_pair_once() -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py new file mode 100644 index 0000000000..cb48b2ee72 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py @@ -0,0 +1,1230 @@ +"""US ESI premium stage tests (microcosm #454). + +Invariants the stage and its gates must hold for every valid input: + +* **Conservation**: the weighted employer premium total equals the BEA NIPA + 7.8 line 17 anchor of the build year, whatever the frame. +* **Bounds**: both outputs are finite and nonnegative; a pre-tax premium is + either zero or the person's reported premium. +* **Structural zeros**: no employer premium outside employed policyholders + with an employer, nor where the employer pays none; no pre-tax premium + outside eligible workers. +* **Proportionality**: every employer premium is the same multiple of its + MEPS-IC cell share. +* **Determinism and order invariance**: values depend on the person, the + frame's weighted raw mass and the seed, never on row order; support clones + agree. +* **Weight homogeneity**: scaling every weight by ``c`` leaves the total at + the anchor and scales each person's value by ``1 / c``. + +A scalar, per-person reference implementation of the declared rules is run +against the vectorized stage on generated frames (differential test). +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +from importlib.resources import files + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from microcosm.build.source_runtime import SourceRuntimeError +from microcosm.build.stochastic_assignment import stable_identity_uniforms +from microcosm.build.us_runtime import ( + US_ESI_EMPLOYER_PREMIUM_COLUMN, + US_ESI_PRE_TAX_PREMIUM_COLUMN, + US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, + US_ESI_PREMIUMS_OUTPUT_COLUMNS, + US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, + US_ESI_PREMIUMS_STAGE_NAME, + US_SOURCE_MANIFEST, + derive_us_employer_esi_premiums_from_manifest, + derive_us_pre_tax_health_insurance_premiums_from_manifest, + load_meps_ic_esi_premium_cells, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, + us_esi_premiums_stage_spec, + us_esi_premiums_summary, + with_us_esi_premium_inputs, +) +from microcosm.build.us_runtime import esi_premiums as esi +from microcosm.build.us_runtime.asec_census_person_columns import ( + ASEC_CENSUS_PERSON_COLUMNS, +) +from microcosm.build.us_runtime.release_input_coverage import ( + us_release_input_coverage_required_columns, + us_release_input_coverage_reviewed_exclusions, + us_release_reform_coverage_probes, +) +from microcosm.build.us_runtime.source_runtime import us_source_operation_handlers +from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights +from test_support.paths import paths_for + +TIME_PERIOD = 2024 +ANCHOR = 977_034e6 +EMPLOYER = US_ESI_EMPLOYER_PREMIUM_COLUMN +PRE_TAX = US_ESI_PRE_TAX_PREMIUM_COLUMN +_REPOSITORY_ROOT = paths_for("microcosm-build").repository +_CELLS = load_meps_ic_esi_premium_cells() +_STATES = sorted(int(state) for state in _CELLS["state_census_division"]) + +#: An employed private-sector self-only policyholder in a large California +#: firm whose employer pays some of the premium, with wages and a premium. +_BASE_ROW = { + "NOW_OWNGRP": 1, + "NOW_HIPAID": 2, + "NOW_GRPFTYP": 2, + "NOW_GRPFTYP2": 3, + "PEMLR": 1, + "NOEMP": 6, + "PEIO1COW": 4, + "PHIP_VAL": 2_000, + "WSAL_VAL": 60_000, + "state_fips": 6, +} +_NO_COVERAGE = {"NOW_OWNGRP": 2, "NOW_HIPAID": 0, "NOW_GRPFTYP": 0, "NOW_GRPFTYP2": 0} + + +def _frame( + rows: list[dict], + *, + weights: list[float] | np.ndarray | None = None, + state_on_household: bool = True, +) -> Frame: + """One household per person; State lives on the household table.""" + + records = [] + for index, row in enumerate(rows): + record = dict(_BASE_ROW) + record.update(row) + record.setdefault("person_id", index + 1) + record.setdefault("source_year", 2024) + record.setdefault("source_household_id", index + 1) + record.setdefault("source_person_id", 1) + records.append(record) + person = pd.DataFrame(records) + count = len(person) + ids = np.arange(1, count + 1, dtype="int64") + person["person_household_id"] = ids + person["person_tax_unit_id"] = ids + 1_000_000 + person["person_spm_unit_id"] = ids + 2_000_000 + person["person_family_id"] = ids + 3_000_000 + person["person_marital_unit_id"] = ids + 4_000_000 + household = pd.DataFrame({"household_id": ids}) + if state_on_household: + household["state_fips"] = person.pop("state_fips").to_numpy() + values = np.ones(count) if weights is None else np.asarray(weights, dtype=float) + return Frame( + { + "person": person, + "household": household, + "tax_unit": pd.DataFrame({"tax_unit_id": ids + 1_000_000}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids + 2_000_000}), + "family": pd.DataFrame({"family_id": ids + 3_000_000}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids + 4_000_000}), + }, + US_SCHEMA, + {"household": Weights(values=values, kind=WeightKind.DESIGN)}, + ) + + +def _run(rows, *, weights=None, seed: int = 0, time_period: int = TIME_PERIOD) -> Frame: + return with_us_esi_premium_inputs( + _frame(rows, weights=weights), seed=seed, time_period=time_period + ) + + +def _values(frame: Frame, column: str) -> np.ndarray: + return frame.table("person")[column].to_numpy(dtype=float) + + +def _weights(frame: Frame) -> np.ndarray: + return np.asarray(frame.resolve_weights("person").values, dtype=float) + + +def _private(tier: str, measure: str, state: int, size: str) -> float: + return _CELLS["private_state_2025"][tier][measure]["rows"][f"{state:02d}"][size] + + +def _realistic_rows(count: int = 400) -> list[dict]: + """A mix whose weighted shares sit inside the signal-gate bands.""" + + rows: list[dict] = [] + for index in range(count): + state = _STATES[index % len(_STATES)] + kind = index % 20 + if kind < 4: # 20% employed policyholders, employer pays some + tier = (index // 20) % 3 + 1 + rows.append( + { + "NOW_GRPFTYP2": tier, + "NOW_GRPFTYP": 1 if tier in (1, 2) else 2, + "NOEMP": (index // 7) % 7, + "PEIO1COW": (1, 2, 3, 4, 5, 6)[(index // 3) % 6], + "PHIP_VAL": 1_000 + 37 * index, + "state_fips": state, + } + ) + elif kind == 4: # employer pays all + rows.append({"NOW_HIPAID": 1, "PHIP_VAL": 0, "state_fips": state}) + elif kind == 5: # retiree policyholder: no wages concept + rows.append({"PEMLR": 5, "PEIO1COW": 0, "NOEMP": 0, "state_fips": state}) + else: # no employment-based coverage of their own + rows.append( + { + **_NO_COVERAGE, + "PEMLR": (0, 1, 5, 7)[index % 4], + "state_fips": state, + } + ) + return rows + + +# --- the manifest contract --------------------------------------------------- + + +class TestManifestDeclaration: + def test_stage_declares_both_outputs_and_the_reviewed_operations(self) -> None: + spec = us_esi_premiums_stage_spec() + assert spec.stage == US_ESI_PREMIUMS_STAGE_NAME == "meps_esi_premiums" + assert tuple(spec.outputs) == US_ESI_PREMIUMS_OUTPUT_COLUMNS + assert US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS == (EMPLOYER, PRE_TAX) + assert [operation.kind for operation in spec.operations] == [ + "read_table", + "derive_employer_sponsored_insurance_premiums", + "derive_pre_tax_health_insurance_premiums", + ] + assert spec is not None and US_SOURCE_MANIFEST.stage_map()[spec.stage] + + def test_handlers_are_registered(self) -> None: + handlers = us_source_operation_handlers() + assert ( + handlers["derive_employer_sponsored_insurance_premiums"] + is derive_us_employer_esi_premiums_from_manifest + ) + assert ( + handlers["derive_pre_tax_health_insurance_premiums"] + is derive_us_pre_tax_health_insurance_premiums_from_manifest + ) + + def test_manifest_pins_the_cells_the_anchor_and_every_source_pdf(self) -> None: + spec = us_esi_premiums_stage_spec() + employer = spec.operations[1].parameters + assert employer["cells_sha256"] == esi._CELLS_SHA256 + assert employer["anchor"]["series"] == "B4923C" + assert employer["anchor"]["values"]["2024"] == ANCHOR + pinned = { + artifact["locator"]: artifact["sha256"] + for artifact in spec.artifacts + if artifact.get("sha256") + } + for source in _CELLS["sources"].values(): + assert pinned[source["url"]] == source["sha256"] + assert ( + pinned[esi.EMPLOYER_PREMIUM_ANCHOR["source"]] + == (esi.EMPLOYER_PREMIUM_ANCHOR["sha256"]) + ) + + def test_a_drifted_manifest_operation_is_refused(self, monkeypatch) -> None: + monkeypatch.setattr( + esi, + "_PRE_TAX_PARAMETERS", + {**esi._PRE_TAX_PARAMETERS, "employer_payment_codes": [2]}, + ) + with pytest.raises(ValueError, match="drifted"): + us_esi_premiums_stage_spec() + + def test_the_restored_census_columns_are_the_ones_the_stage_requires(self) -> None: + restored = {column.name: column for column in ASEC_CENSUS_PERSON_COLUMNS} + for name in ("NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2"): + assert restored[name].domain == esi._CODE_DOMAINS[name] + assert restored["PEMLR"].domain == esi._CODE_DOMAINS["PEMLR"] + assert restored["NOEMP"].domain == esi._CODE_DOMAINS["NOEMP"] + assert set(restored) >= set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) - { + "PEIO1COW", + "PHIP_VAL", + "WSAL_VAL", + "state_fips", + } + + +# --- the packaged MEPS-IC cells ----------------------------------------------- + + +class TestCells: + def test_digest_is_pinned(self, monkeypatch) -> None: + monkeypatch.setattr(esi, "_CELLS_SHA256", "0" * 64) + with pytest.raises(SourceRuntimeError, match="not the reviewed"): + load_meps_ic_esi_premium_cells() + + def test_national_rows_are_the_published_ahrq_figures(self) -> None: + rows = _CELLS["private_state_2025"] + assert rows["single"]["premium"]["rows"]["US"]["total"] == 9_025 + assert rows["family"]["premium"]["rows"]["US"]["total"] == 26_281 + assert rows["employee_plus_one"]["premium"]["rows"]["US"]["total"] == 17_901 + prior = _CELLS["private_national_2024"] + # MEPS-IC Research Findings #54 (2024). + assert prior["single"]["premium"]["total"] == 8_486 + assert prior["single"]["employee_contribution"]["total"] == 1_789 + assert prior["family"]["premium"]["total"] == 24_540 + assert prior["family"]["employee_contribution"]["total"] == 7_216 + assert prior["employee_plus_one"]["premium"]["total"] == 16_931 + assert prior["employee_plus_one"]["employee_contribution"]["total"] == 4_707 + + def test_every_state_and_division_has_a_cell(self) -> None: + divisions = _CELLS["state_census_division"] + assert len(divisions) == 51 + assert len(set(divisions.values())) == 9 + for tier in _CELLS["tiers"]: + for measure in ("premium", "employee_contribution"): + rows = _CELLS["private_state_2025"][tier][measure]["rows"] + assert set(rows) == {"US", *divisions} + public = _CELLS["public_division_2024"][tier][measure]["rows"] + assert set(public) == {"US", *set(divisions.values())} + + def test_suppressed_cells_are_the_nine_small_firm_contributions(self) -> None: + suppressed = { + (tier, measure, state, size) + for tier in _CELLS["tiers"] + for measure in ("premium", "employee_contribution") + for state, row in _CELLS["private_state_2025"][tier][measure][ + "rows" + ].items() + for size in ("total", "lt50", "50plus") + if row[size] is None + } + assert suppressed == { + ("employee_plus_one", "employee_contribution", state, "lt50") + for state in ("02", "21", "45", "46") + } | { + ("family", "employee_contribution", state, "lt50") + for state in ("02", "10", "35", "45", "54") + } + + def test_every_resolved_cell_leaves_a_positive_employer_share(self) -> None: + for tier in _CELLS["tiers"]: + for state in _CELLS["state_census_division"]: + for size in ("total", "lt50", "50plus"): + premium = esi._private_cell(_CELLS, tier, "premium", state, size) + paid = esi._private_cell( + _CELLS, tier, "employee_contribution", state, size + ) + assert 0 < paid < premium < 60_000 + for division in set(_CELLS["state_census_division"].values()): + for column in ("state", "all_state_and_local"): + premium = esi._government_cell( + _CELLS, tier, "premium", division, column + ) + paid = esi._government_cell( + _CELLS, tier, "employee_contribution", division, column + ) + assert 0 < paid < premium < 60_000 + + def test_pre_tax_shares_are_offer_rate_ratios(self) -> None: + shares = esi._pre_tax_shares(_CELLS) + assert shares == pytest.approx( + {"lt50": 15.9 / 32.1, "50plus": 87.8 / 94.8, "total": 35.5 / 49.2} + ) + assert all(0 < share < 1 for share in shares.values()) + + +# --- the employer premium ------------------------------------------------------- + + +class TestEmployerPremium: + def test_weighted_total_is_the_bea_anchor(self) -> None: + result = _run(_realistic_rows(), weights=np.linspace(50, 5_000, 400)) + total = _weights(result) @ _values(result, EMPLOYER) + assert total == pytest.approx(ANCHOR, rel=1e-12) + + def test_payment_status_sets_the_share_of_the_cell(self) -> None: + rows = [{"NOW_HIPAID": code, "PHIP_VAL": 500} for code in (1, 2, 3)] + employer = _values(_run(rows), EMPLOYER) + premium = _private("single", "premium", 6, "50plus") + paid = _private("single", "employee_contribution", 6, "50plus") + assert employer[2] == 0 + assert employer[0] / employer[1] == pytest.approx(premium / (premium - paid)) + # One scale factor: both are the same multiple of their raw share. + assert employer[0] / premium == pytest.approx(employer[1] / (premium - paid)) + + def test_only_employed_policyholders_with_an_employer_carry_a_premium(self) -> None: + rows = [ + {}, # the universe + dict(_NO_COVERAGE), # dependent or uncovered + {"NOW_OWNGRP": 0, "NOW_HIPAID": 0, "NOW_GRPFTYP": 0, "NOW_GRPFTYP2": 0}, + {"PEMLR": 5, "PEIO1COW": 0}, # retiree policyholder + {"PEMLR": 3}, # on layoff + {"PEMLR": 4}, # looking + {"PEMLR": 6}, # disabled + {"PEIO1COW": 7}, # self-employed, unincorporated + {"PEIO1COW": 8}, # without pay + {"PEMLR": 2}, # employed, absent + {"PEIO1COW": 6}, # self-employed, incorporated + ] + employer = _values(_run(rows), EMPLOYER) + assert (employer[[0, 9, 10]] > 0).all() + assert (employer[1:9] == 0).all() + + @pytest.mark.parametrize( + ("grpftyp", "grpftyp2", "tier"), + [(1, 1, "family"), (1, 2, "employee_plus_one"), (2, 3, "single")], + ) + def test_tier_selects_the_cell(self, grpftyp, grpftyp2, tier) -> None: + rows = [ + {"NOW_HIPAID": 1}, + {"NOW_HIPAID": 1, "NOW_GRPFTYP": grpftyp, "NOW_GRPFTYP2": grpftyp2}, + ] + employer = _values(_run(rows), EMPLOYER) + assert employer[1] / employer[0] == pytest.approx( + _private(tier, "premium", 6, "50plus") + / _private("single", "premium", 6, "50plus") + ) + + @pytest.mark.parametrize( + ("noemp", "size"), + [(0, "total"), (1, "lt50"), (2, "lt50"), (3, "50plus"), (5, "50plus")], + ) + def test_employer_size_and_state_select_the_private_cell(self, noemp, size) -> None: + rows = [{"NOW_HIPAID": 1}, {"NOW_HIPAID": 1, "NOEMP": noemp, "state_fips": 36}] + employer = _values(_run(rows), EMPLOYER) + assert employer[1] / employer[0] == pytest.approx( + _private("single", "premium", 36, size) + / _private("single", "premium", 6, "50plus") + ) + + @pytest.mark.parametrize( + ("sector", "column"), + [(1, "all_state_and_local"), (2, "state"), (3, "all_state_and_local")], + ) + def test_government_employers_take_the_aged_division_cell( + self, sector, column + ) -> None: + # Texas is West South Central; NOEMP must not matter for government. + rows = [ + {"NOW_HIPAID": 1}, + {"NOW_HIPAID": 1, "PEIO1COW": sector, "state_fips": 48, "NOEMP": 1}, + ] + employer = _values(_run(rows), EMPLOYER) + assert _CELLS["state_census_division"]["48"] == "West South Central" + published = _CELLS["public_division_2024"]["single"]["premium"]["rows"][ + "West South Central" + ][column] + aged = published * 9_025 / 8_486 + assert employer[1] / employer[0] == pytest.approx( + aged / _private("single", "premium", 6, "50plus") + ) + + def test_a_suppressed_cell_takes_the_national_cell_at_the_state_level(self) -> None: + # Alaska family small-firm employee contribution is suppressed. + family = {"NOW_GRPFTYP": 1, "NOW_GRPFTYP2": 1, "state_fips": 2, "NOEMP": 1} + rows = [{**family, "NOW_HIPAID": 1}, {**family, "NOW_HIPAID": 2}] + employer = _values(_run(rows), EMPLOYER) + contributions = _CELLS["private_state_2025"]["family"]["employee_contribution"][ + "rows" + ] + assert contributions["02"]["lt50"] is None + fallback = ( + contributions["US"]["lt50"] + * contributions["02"]["total"] + / contributions["US"]["total"] + ) + premium = _private("family", "premium", 2, "lt50") + assert employer[1] / employer[0] == pytest.approx( + (premium - fallback) / premium + ) + + def test_state_may_already_sit_on_the_person_table(self) -> None: + rows = _realistic_rows(60) + on_household = with_us_esi_premium_inputs( + _frame(rows), seed=0, time_period=TIME_PERIOD + ) + on_person = with_us_esi_premium_inputs( + _frame(rows, state_on_household=False), seed=0, time_period=TIME_PERIOD + ) + assert np.array_equal( + _values(on_household, EMPLOYER), _values(on_person, EMPLOYER) + ) + + @pytest.mark.parametrize("year", [2023, 2025]) + def test_each_pinned_year_scales_to_its_own_anchor(self, year) -> None: + result = _run(_realistic_rows(60), time_period=year) + total = _weights(result) @ _values(result, EMPLOYER) + assert total == pytest.approx( + esi.EMPLOYER_PREMIUM_ANCHOR["values"][str(year)], rel=1e-12 + ) + + +class TestRefusals: + @pytest.mark.parametrize( + "column", [c for c in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS] + ) + def test_a_missing_source_column_is_source_unavailable(self, column) -> None: + frame = _frame(_realistic_rows(40), state_on_household=False) + frame.table("person").drop(columns=[column], inplace=True) + with pytest.raises(SourceRuntimeError, match="source-unavailable"): + with_us_esi_premium_inputs(frame, seed=0, time_period=TIME_PERIOD) + + @pytest.mark.parametrize("column", ["NOW_OWNGRP", "NOW_HIPAID", "PEMLR", "NOEMP"]) + def test_a_pooled_nan_is_never_defaulted(self, column) -> None: + frame = _frame(_realistic_rows(40)) + person = frame.table("person") + person[column] = person[column].astype(float) + person.loc[3, column] = np.nan + with pytest.raises(SourceRuntimeError, match="missing or non-integer"): + with_us_esi_premium_inputs(frame, seed=0, time_period=TIME_PERIOD) + + @pytest.mark.parametrize( + ("column", "value"), + [("NOW_OWNGRP", 3), ("NOW_HIPAID", 4), ("NOW_GRPFTYP2", 4), ("NOEMP", 7)], + ) + def test_a_code_outside_the_census_codebook_is_refused(self, column, value) -> None: + with pytest.raises(SourceRuntimeError, match="outside the Census codebook"): + _run([{}, {column: value}]) + + @pytest.mark.parametrize( + ("row", "violation"), + [ + ({"NOW_HIPAID": 0}, "policyholder_without_payment_status"), + ({"NOW_GRPFTYP2": 0}, "policyholder_without_tier"), + ({**_NO_COVERAGE, "NOW_HIPAID": 2}, "non_policyholder_with_payment"), + ({**_NO_COVERAGE, "NOW_GRPFTYP2": 3}, "non_policyholder_with_tier"), + ({"NOW_GRPFTYP": 1, "NOW_GRPFTYP2": 3}, "grpftyp_disagrees"), + ], + ) + def test_the_census_universe_relations_are_enforced(self, row, violation) -> None: + with pytest.raises(SourceRuntimeError, match=violation): + _run([{}, row]) + + def test_a_frame_with_no_employer_paid_mass_cannot_be_scaled(self) -> None: + with pytest.raises(SourceRuntimeError, match="no weighted employer-paid"): + _run([{"NOW_HIPAID": 3}, dict(_NO_COVERAGE)]) + + def test_an_unpinned_build_year_is_refused(self) -> None: + with pytest.raises(SourceRuntimeError, match="no BEA anchor for 2030"): + _run([{}], time_period=2030) + + def test_an_unknown_state_is_refused(self) -> None: + with pytest.raises(SourceRuntimeError, match="State FIPS 72"): + _run([{}, {"state_fips": 72}]) + + +# --- the pre-tax employee premium ------------------------------------------------ + + +class TestPreTaxPremium: + def test_only_eligible_workers_can_pay_pre_tax(self) -> None: + eligible = [{"person_id": index + 1} for index in range(300)] + ineligible = [ + {"NOW_HIPAID": 1}, # the employer pays everything + {"WSAL_VAL": 0}, # no wages to deduct from + {"PHIP_VAL": 0}, # no premium reported + {"PEMLR": 5, "PEIO1COW": 0}, # not employed + {"PEIO1COW": 7}, # no employer + dict(_NO_COVERAGE), # not a policyholder + ] + result = _run(eligible + ineligible) + pre_tax = _values(result, PRE_TAX) + assert (pre_tax[300:] == 0).all() + assert set(np.unique(pre_tax[:300])) == {0.0, 2_000.0} + + @pytest.mark.parametrize( + ("noemp", "size"), [(0, "total"), (1, "lt50"), (2, "lt50"), (6, "50plus")] + ) + def test_selection_rate_is_the_firm_size_offer_ratio(self, noemp, size) -> None: + # Both payment statuses that leave the worker a share to pay. + rows = [{"NOEMP": noemp, "NOW_HIPAID": 2 + index % 2} for index in range(6_000)] + pre_tax = _values(_run(rows, seed=11), PRE_TAX) + share = esi._pre_tax_shares(_CELLS)[size] + # Six thousand Bernoulli draws: five standard errors is under 3.3 points. + assert (pre_tax > 0).mean() == pytest.approx(share, abs=0.033) + + def test_draws_are_the_shared_identity_keyed_uniforms(self) -> None: + rows = [{"source_household_id": 900 + index} for index in range(50)] + pre_tax = _values(_run(rows, seed=5), PRE_TAX) + draws = stable_identity_uniforms( + [f"2024:{900 + index}:1" for index in range(50)], + seed=5, + salt=PRE_TAX, + ) + expected = np.where(draws < esi._pre_tax_shares(_CELLS)["50plus"], 2_000.0, 0.0) + assert np.array_equal(pre_tax, expected) + + def test_the_seed_changes_who_is_selected_not_the_employer_premium(self) -> None: + rows = [{"NOEMP": 1} for _ in range(400)] + first, second = _run(rows, seed=1), _run(rows, seed=2) + assert not np.array_equal(_values(first, PRE_TAX), _values(second, PRE_TAX)) + assert np.array_equal(_values(first, EMPLOYER), _values(second, EMPLOYER)) + + +# --- idempotence and support clones ----------------------------------------------- + + +def _cloned(frame: Frame) -> Frame: + """Two support clones of every person at half weight, same source identity.""" + + person = frame.table("person") + doubled = pd.concat([person, person], ignore_index=True) + count = len(person) + doubled["person_id"] = np.arange(1, 2 * count + 1) + for column, offset in ( + ("person_household_id", 0), + ("person_tax_unit_id", 1_000_000), + ("person_spm_unit_id", 2_000_000), + ("person_family_id", 3_000_000), + ("person_marital_unit_id", 4_000_000), + ): + doubled[column] = np.arange(1, 2 * count + 1) + offset + ids = np.arange(1, 2 * count + 1, dtype="int64") + household = pd.concat( + [frame.table("household"), frame.table("household")], ignore_index=True + ) + household["household_id"] = ids + weights = np.tile(frame.weights_for("household").values / 2.0, 2) + return Frame( + { + "person": doubled, + "household": household, + "tax_unit": pd.DataFrame({"tax_unit_id": ids + 1_000_000}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids + 2_000_000}), + "family": pd.DataFrame({"family_id": ids + 3_000_000}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids + 4_000_000}), + }, + US_SCHEMA, + {"household": Weights(values=weights, kind=WeightKind.DESIGN)}, + ) + + +class TestIdempotenceAndClones: + def test_a_frame_that_carries_the_inputs_passes_through_untouched(self) -> None: + once = _run(_realistic_rows()) + twice = with_us_esi_premium_inputs(once, seed=99, time_period=TIME_PERIOD) + assert twice is once + + def test_support_clones_conserve_mass_and_agree(self) -> None: + once = _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) + cloned = _cloned(once) + assert us_esi_premiums_signal_gate(cloned).passed + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + assert _weights(cloned) @ _values(cloned, column) == pytest.approx( + _weights(once) @ _values(once, column), rel=1e-12 + ) + assert us_esi_premiums_anchor_gate(cloned, time_period=TIME_PERIOD).passed + + def test_a_preexisting_surface_that_fails_the_gate_is_refused(self) -> None: + once = _run(_realistic_rows()) + person = once.table("person") + holder = int(np.flatnonzero(person[EMPLOYER].to_numpy() > 0)[0]) + person.loc[holder, EMPLOYER] *= 2 + with pytest.raises(SourceRuntimeError, match="fail the signal gate"): + with_us_esi_premium_inputs(once, seed=0, time_period=TIME_PERIOD) + + +# --- the stage signal gate -------------------------------------------------------- + + +class TestSignalGate: + def _staged(self) -> Frame: + return _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) + + def test_passes_on_a_staged_frame_and_reports_the_lineage(self) -> None: + gate = us_esi_premiums_signal_gate(self._staged()) + assert gate.passed, gate.failures + details = gate.details + assert details["employer_premium_total"] == pytest.approx(ANCHOR, rel=1e-12) + assert details["scale_factor"] == pytest.approx( + ANCHOR / details["raw_employer_share_total"] + ) + assert details["scale_factor_spread"] <= 1e-9 * details["scale_factor"] + assert details["cells_sha256"] == esi._CELLS_SHA256 + assert details["anchor"]["series"] == "B4923C" + assert sum(details["employer_premium_by_tier"].values()) == pytest.approx( + ANCHOR + ) + assert sum(details["employer_premium_by_sector"].values()) == pytest.approx( + ANCHOR + ) + json.dumps(dict(details)) # the base summary serializes it + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_missing_column_fails(self, column) -> None: + frame = self._staged() + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert gate.failures == (f"person column missing: {column}.",) + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_zero_mass_column_fails(self, column) -> None: + frame = self._staged() + frame.table("person")[column] = 0.0 + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert any("constant column" in failure for failure in gate.failures) + + def _fails_with(self, mutate, match: str) -> None: + frame = self._staged() + mutate(frame.table("person")) + gate = us_esi_premiums_signal_gate(frame) + assert not gate.passed + assert any(match in failure for failure in gate.failures), gate.failures + + def test_a_premium_outside_the_universe_fails(self) -> None: + def mutate(person): + outsider = int(np.flatnonzero(person["NOW_OWNGRP"].to_numpy() == 2)[0]) + person.loc[outsider, EMPLOYER] = 9_000.0 + + self._fails_with(mutate, "outside employed policyholders") + + def test_a_premium_where_the_employer_pays_none_fails(self) -> None: + frame = _run(_realistic_rows() + [{"NOW_HIPAID": 3}]) + frame.table("person").loc[400, EMPLOYER] = 5_000.0 + gate = us_esi_premiums_signal_gate(frame) + assert any("employer pays none" in failure for failure in gate.failures) + + def test_a_value_that_is_not_the_common_cell_multiple_fails(self) -> None: + def mutate(person): + holder = int(np.flatnonzero(person[EMPLOYER].to_numpy() > 0)[0]) + person.loc[holder, EMPLOYER] *= 1.01 + + self._fails_with(mutate, "not one common multiple") + + def test_a_negative_value_fails(self) -> None: + self._fails_with( + lambda person: person.__setitem__(PRE_TAX, -person[PRE_TAX]), + "negative value", + ) + + def test_a_pre_tax_premium_for_an_ineligible_person_fails(self) -> None: + def mutate(person): + outsider = int(np.flatnonzero(person["NOW_OWNGRP"].to_numpy() == 2)[0]) + person.loc[outsider, PRE_TAX] = float(person.loc[outsider, "PHIP_VAL"]) + + self._fails_with(mutate, "ineligible people") + + def test_a_pre_tax_premium_that_is_not_the_reported_premium_fails(self) -> None: + def mutate(person): + payer = int(np.flatnonzero(person[PRE_TAX].to_numpy() > 0)[0]) + person.loc[payer, PRE_TAX] += 1.0 + + self._fails_with(mutate, "not equal to PHIP_VAL") + + def test_clone_disagreement_fails(self) -> None: + cloned = _cloned(self._staged()) + person = cloned.table("person") + holder = int(np.flatnonzero(person[EMPLOYER].to_numpy() > 0)[0]) + person.loc[holder, EMPLOYER] = 0.0 + gate = us_esi_premiums_signal_gate(cloned) + assert any("support-clone disagreement" in f for f in gate.failures) + + def test_an_implausible_positive_share_fails(self) -> None: + gate = us_esi_premiums_signal_gate(_run([{} for _ in range(50)])) + assert any("positive share" in failure for failure in gate.failures) + + def test_a_broken_cell_table_cannot_hide_behind_the_scale_factor( + self, monkeypatch + ) -> None: + broken = json.loads(json.dumps(_CELLS)) + for tier in broken["tiers"]: + for measure in ("premium", "employee_contribution"): + for row in broken["private_state_2025"][tier][measure]["rows"].values(): + for size in ("total", "lt50", "50plus"): + if row[size] is not None: + row[size] /= 100.0 + for row in broken["public_division_2024"][tier][measure][ + "rows" + ].values(): + for column in row: + row[column] /= 100.0 + monkeypatch.setattr(esi, "load_meps_ic_esi_premium_cells", lambda: broken) + frame = _run(_realistic_rows()) + total = _weights(frame) @ _values(frame, EMPLOYER) + assert total == pytest.approx(ANCHOR) # the scale factor hid the level + gate = us_esi_premiums_signal_gate(frame) + assert any("the cell table is broken" in f for f in gate.failures) + + def test_an_export_frame_without_raw_columns_is_still_graded(self) -> None: + frame = self._staged() + frame.table("person").drop( + columns=[ + column + for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column != "state_fips" + ], + inplace=True, + ) + gate = us_esi_premiums_signal_gate(frame) + assert gate.passed, gate.failures + assert "scale_factor" not in gate.details + frame.table("person")[EMPLOYER] = 0.0 + assert not us_esi_premiums_signal_gate(frame).passed + + +# --- the release anchor gate -------------------------------------------------------- + + +class TestAnchorGate: + def _export(self, *, scale: float = 1.0, pre_tax_scale: float = 1.0) -> Frame: + """A calibrated export: engine inputs only, reweighted by ``scale``.""" + + frame = _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) + person = frame.table("person") + person.drop( + columns=list( + set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) & set(person.columns) + ), + inplace=True, + ) + person[EMPLOYER] *= scale + person[PRE_TAX] *= pre_tax_scale + return frame + + @pytest.mark.parametrize("scale", [0.951, 1.0, 1.049]) + def test_passes_inside_the_stated_tolerance(self, scale) -> None: + gate = us_esi_premiums_anchor_gate( + self._export(scale=scale), time_period=TIME_PERIOD + ) + assert gate.passed, gate.failures + assert gate.details["relative_error"] == pytest.approx(scale - 1.0) + assert gate.details["anchor_value"] == ANCHOR + assert gate.details["relative_tolerance"] == esi.ANCHOR_RELATIVE_TOLERANCE + + @pytest.mark.parametrize("scale", [0.94, 1.06]) + def test_fails_outside_the_stated_tolerance(self, scale) -> None: + gate = us_esi_premiums_anchor_gate( + self._export(scale=scale), time_period=TIME_PERIOD + ) + assert not gate.passed + assert any("from BEA NIPA 7.8 line 17" in f for f in gate.failures) + + def test_the_retiree_inclusive_nhe_total_is_an_upper_bound(self) -> None: + # NHE is 7.2% above BEA, so the tolerance fails first; the bound must + # also be named for a release that drifted to the NHE concept. + gate = us_esi_premiums_anchor_gate( + self._export(scale=1.08), time_period=TIME_PERIOD + ) + assert any("exceeds the retiree-inclusive NHE" in f for f in gate.failures) + assert esi.ANCHOR_RELATIVE_TOLERANCE < 1_047.0e9 / ANCHOR - 1.0 + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_an_absent_column_is_red(self, column) -> None: + frame = self._export() + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not gate.passed + assert gate.failures == (f"person column missing: {column}.",) + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_zero_mass_column_is_red(self, column) -> None: + frame = self._export() + frame.table("person")[column] = 0.0 + gate = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not gate.passed + assert f"{column}: zero weighted mass." in gate.failures + + def test_pre_tax_premiums_cannot_exceed_nhe_employee_contributions(self) -> None: + frame = self._export() + total = _weights(frame) @ _values(frame, PRE_TAX) + gate = us_esi_premiums_anchor_gate( + self._export(pre_tax_scale=1.01 * 382.1e9 / total), + time_period=TIME_PERIOD, + ) + assert any("employee contribution" in f for f in gate.failures) + + def test_a_year_without_an_anchor_is_red(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=2031) + assert not gate.passed + assert any( + "no BEA NIPA 7.8 line 17 anchor for 2031" in f for f in gate.failures + ) + + def test_details_serialize_for_the_release_evidence_file(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) + payload = json.loads(json.dumps(dict(gate.details))) + assert payload["anchor"]["sha256"] == esi.EMPLOYER_PREMIUM_ANCHOR["sha256"] + assert payload["cross_check"]["employer_contribution"]["2024"] == 1_047.0e9 + + +# --- properties and the differential reference ------------------------------------ + + +@st.composite +def _person_rows(draw) -> dict: + owner = draw(st.sampled_from([0, 1, 1, 2])) + if owner == 1: + tier = draw(st.sampled_from([1, 2, 3])) + coverage = { + "NOW_HIPAID": draw(st.sampled_from([1, 2, 2, 3])), + "NOW_GRPFTYP2": tier, + "NOW_GRPFTYP": 1 if tier in (1, 2) else 2, + } + else: + coverage = {"NOW_HIPAID": 0, "NOW_GRPFTYP2": 0, "NOW_GRPFTYP": 0} + return { + "NOW_OWNGRP": owner, + **coverage, + "PEMLR": draw(st.sampled_from([0, 1, 1, 1, 2, 3, 4, 5, 6, 7])), + "NOEMP": draw(st.integers(0, 6)), + "PEIO1COW": draw(st.sampled_from([0, 1, 2, 3, 4, 4, 4, 5, 6, 7, 8])), + "PHIP_VAL": draw(st.sampled_from([0, 0, 350, 1_800, 7_400, 22_000])), + "WSAL_VAL": draw(st.sampled_from([0, 12_000, 48_000, 250_000])), + "state_fips": draw(st.sampled_from(_STATES)), + } + + +@st.composite +def _populations(draw): + rows = draw(st.lists(_person_rows(), min_size=1, max_size=40)) + # At least one person with employer-paid mass, so the frame can be scaled. + rows.append( + { + "NOW_HIPAID": draw(st.sampled_from([1, 2])), + "state_fips": draw(st.sampled_from(_STATES)), + "PEIO1COW": draw(st.sampled_from([1, 2, 3, 4, 5, 6])), + } + ) + weights = draw( + st.lists( + st.floats(0.25, 9_000.0, allow_nan=False), + min_size=len(rows), + max_size=len(rows), + ) + ) + return rows, weights + + +def _reference(rows: list[dict], weights: list[float], *, seed: int): + """A scalar restatement of the declared rules, one person at a time.""" + + tiers = {1: "family", 2: "employee_plus_one", 3: "single"} + raw: list[float] = [] + pre_tax: list[float] = [] + for index, partial in enumerate(rows): + row = {**_BASE_ROW, **partial} + in_universe = ( + row["NOW_OWNGRP"] == 1 + and row["PEMLR"] in (1, 2) + and row["PEIO1COW"] in (1, 2, 3, 4, 5, 6) + ) + size = ( + "total" + if row["NOEMP"] == 0 + else "lt50" + if row["NOEMP"] in (1, 2) + else "50plus" + ) + share = 0.0 + if in_universe and row["NOW_HIPAID"] in (1, 2): + tier = tiers[row["NOW_GRPFTYP2"]] + state = f"{row['state_fips']:02d}" + cell = {} + for measure in ("premium", "employee_contribution"): + private = _CELLS["private_state_2025"][tier][measure]["rows"] + if row["PEIO1COW"] in (4, 5, 6): + value = private[state][size] + if value is None: + value = ( + private["US"][size] + * private[state]["total"] + / private["US"]["total"] + ) + else: + division = _CELLS["state_census_division"][state] + column = "state" if row["PEIO1COW"] == 2 else "all_state_and_local" + value = ( + _CELLS["public_division_2024"][tier][measure]["rows"][division][ + column + ] + * private["US"]["total"] + / _CELLS["private_national_2024"][tier][measure]["total"] + ) + cell[measure] = value + share = ( + cell["premium"] + if row["NOW_HIPAID"] == 1 + else max(cell["premium"] - cell["employee_contribution"], 0.0) + ) + raw.append(share) + eligible = ( + in_universe + and row["NOW_HIPAID"] in (2, 3) + and row["WSAL_VAL"] > 0 + and row["PHIP_VAL"] > 0 + ) + draw = stable_identity_uniforms( + [f"2024:{index + 1}:1"], seed=seed, salt=PRE_TAX + )[0] + offer = _CELLS["pretax_contribution_2025"]["rows"][size] + probability = ( + offer["pretax_contribution_offer_percent"] + / offer["health_insurance_offer_percent"] + ) + pre_tax.append( + float(row["PHIP_VAL"]) if eligible and draw < probability else 0.0 + ) + total = sum(weight * share for weight, share in zip(weights, raw, strict=True)) + return [share * ANCHOR / total for share in raw], pre_tax + + +_PROPERTY_SETTINGS = settings( + max_examples=120, + deadline=None, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +@_PROPERTY_SETTINGS +@given(_populations(), st.integers(0, 2**31 - 1)) +def test_stage_agrees_with_the_scalar_reference(population, seed) -> None: + rows, weights = population + result = _run(rows, weights=weights, seed=seed) + employer, pre_tax = _reference(rows, weights, seed=seed) + np.testing.assert_allclose(_values(result, EMPLOYER), employer, rtol=1e-11) + assert _values(result, PRE_TAX).tolist() == pre_tax + + +@_PROPERTY_SETTINGS +@given(_populations(), st.integers(0, 2**31 - 1)) +def test_conservation_bounds_and_structural_zeros(population, seed) -> None: + rows, weights = population + result = _run(rows, weights=weights, seed=seed) + person = result.table("person") + employer, pre_tax = _values(result, EMPLOYER), _values(result, PRE_TAX) + assert _weights(result) @ employer == pytest.approx(ANCHOR, rel=1e-11) + assert np.isfinite(employer).all() and np.isfinite(pre_tax).all() + assert (employer >= 0).all() and (pre_tax >= 0).all() + universe = ( + person["NOW_OWNGRP"].eq(1) + & person["PEMLR"].isin([1, 2]) + & person["PEIO1COW"].isin([1, 2, 3, 4, 5, 6]) + ).to_numpy() + assert (employer[~universe] == 0).all() + assert (employer[universe & person["NOW_HIPAID"].eq(3).to_numpy()] == 0).all() + assert (employer[universe & person["NOW_HIPAID"].isin([1, 2]).to_numpy()] > 0).all() + phip = person["PHIP_VAL"].to_numpy(dtype=float) + assert ((pre_tax == 0) | (pre_tax == phip)).all() + eligible = ( + universe + & person["NOW_HIPAID"].isin([2, 3]).to_numpy() + & (person["WSAL_VAL"].to_numpy() > 0) + & (phip > 0) + ) + assert (pre_tax[~eligible] == 0).all() + summary = us_esi_premiums_summary(result) + for key in ( + "nonfinite_rows", + "negative_rows", + "clone_disagreement_source_persons", + "employer_premium_outside_universe_rows", + "employer_premium_where_employer_pays_none_rows", + "employer_premium_without_raw_share_rows", + "pre_tax_ineligible_rows", + "pre_tax_not_reported_premium_rows", + ): + assert summary[key] == 0, key + assert summary["scale_factor_spread"] <= 1e-9 * summary["scale_factor"] + + +@_PROPERTY_SETTINGS +@given(_populations(), st.randoms(use_true_random=False)) +def test_row_order_does_not_change_anyone(population, random) -> None: + rows, weights = population + keyed = [ + {**row, "person_id": index + 1, "source_household_id": index + 1} + for index, row in enumerate(rows) + ] + order = list(range(len(keyed))) + random.shuffle(order) + straight = _run(keyed, weights=weights, seed=3).table("person") + shuffled = _run( + [keyed[index] for index in order], + weights=[weights[index] for index in order], + seed=3, + ).table("person") + straight = straight.set_index("person_id").sort_index() + shuffled = shuffled.set_index("person_id").sort_index() + np.testing.assert_allclose( + shuffled[EMPLOYER].to_numpy(), straight[EMPLOYER].to_numpy(), rtol=1e-11 + ) + assert shuffled[PRE_TAX].tolist() == straight[PRE_TAX].tolist() + + +@_PROPERTY_SETTINGS +@given(_populations(), st.floats(1e-3, 1e3, allow_nan=False)) +def test_weights_are_homogeneous_of_degree_minus_one(population, factor) -> None: + rows, weights = population + base = _run(rows, weights=weights) + scaled = _run(rows, weights=[weight * factor for weight in weights]) + np.testing.assert_allclose( + _values(scaled, EMPLOYER) * factor, _values(base, EMPLOYER), rtol=1e-10 + ) + assert _weights(scaled) @ _values(scaled, EMPLOYER) == pytest.approx( + ANCHOR, rel=1e-11 + ) + assert _values(scaled, PRE_TAX).tolist() == _values(base, PRE_TAX).tolist() + + +@_PROPERTY_SETTINGS +@given(_populations()) +def test_staging_is_idempotent_and_clone_safe(population) -> None: + rows, weights = population + once = _run(rows, weights=weights) + assert ( + us_esi_premiums_summary(_cloned(once))["clone_disagreement_source_persons"] == 0 + ) + cloned = _cloned(once) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + assert _weights(cloned) @ _values(cloned, column) == pytest.approx( + _weights(once) @ _values(once, column), rel=1e-11, abs=1e-6 + ) + + +# --- the registers the stage moves ------------------------------------------------- + + +class TestRegisters: + def test_both_inputs_are_required_release_columns_with_no_exclusion(self) -> None: + required = us_release_input_coverage_required_columns() + assert {EMPLOYER, PRE_TAX} <= required + assert not {EMPLOYER, PRE_TAX} & set( + us_release_input_coverage_reviewed_exclusions() + ) + + def test_the_parity_gap_register_no_longer_exempts_the_employer_premium( + self, + ) -> None: + payload = json.loads( + files("microcosm.build.us") + .joinpath("ecps_parity_known_gaps.json") + .read_text() + ) + assert EMPLOYER not in payload["known_gaps"] + + def test_shipped_reform_probes_bind_through_both_inputs(self) -> None: + probes = {probe.id: probe for probe in us_release_reform_coverage_probes()} + employer = probes["employer_sponsored_insurance_premium_neutralization"] + assert employer.neutralized_variable == EMPLOYER + assert employer.budget_measure == "cbo_household_market_income" + assert employer.effect_direction == "baseline_minus_reform" + assert employer.expected_sign == "positive" + assert 0 < employer.min_abs_effect < ANCHOR + pre_tax = probes["pre_tax_health_insurance_premium_neutralization"] + assert pre_tax.neutralized_variable == PRE_TAX + assert pre_tax.budget_measure == "income_tax" + assert pre_tax.expected_sign == "negative" + + def test_the_nhe_target_family_is_fenced_as_a_broader_concept(self) -> None: + manifest = json.loads( + files("microcosm.build.us") + .joinpath("target_parity_manifest.json") + .read_text() + ) + families = manifest["families"] + for family in ( + "cms_nhe.esi_employer_contribution_premiums", + "cms_nhe.esi_private_employer_contribution_premiums", + ): + entry = families[family] + assert entry["status"] == "reviewed_exclusion" + assert entry["classification"] == "deferred" + assert "retiree" in entry["reason"] + assert "us_esi_premiums_anchor_gate" in entry["reason"] + absent = families["bea_nipa.private_group_health_insurance"] + assert absent["classification"] == "source_absent" + assert "B4923C" in absent["reason"] + + @pytest.mark.parametrize( + ("pool", "scale", "holders", "mean", "pre_tax", "nhe_ratio"), + [ + # The default pool (docs/us-asec-source-pins.md) and the historical + # Build J/N/P pool; figures quoted in the docs and the registers. + ("2023_2025", 1.0703, 73.97e6, 13_209, 181.1e9, 0.985), + ("2022_2024", 1.0664, 74.06e6, 13_193, 174.6e9, 0.990), + ], + ) + def test_the_receipts_reproduce_the_documented_measurements( + self, pool, scale, holders, mean, pre_tax, nhe_ratio + ) -> None: + receipt = json.loads( + ( + _REPOSITORY_ROOT + / f"experiments/us-esi-454/receipts/stage_on_pool_{pool}.json" + ).read_text() + ) + summary = receipt["summary"] + years = [int(year) for year in pool.split("_")] + assert receipt["pooled_income_years"] == list(range(years[0], years[1] + 1)) + assert receipt["signal_gate"] == {"passed": True, "failures": []} + assert receipt["anchor_gate"]["passed"] is True + clone = receipt["puf_support_clone"] + assert clone["signal_gate"]["passed"] and clone["anchor_gate"]["passed"] + assert clone["anchor_gate"]["employer_premium_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + assert summary["employer_premium_total"] == pytest.approx(ANCHOR, rel=1e-12) + assert summary["cells_sha256"] == esi._CELLS_SHA256 + assert summary["scale_factor"] == pytest.approx(scale, abs=5e-5) + assert summary["employer_premium_positive_persons"] == pytest.approx( + holders, rel=1e-3 + ) + assert summary["employer_premium_mean_per_positive_person"] == pytest.approx( + mean, abs=1 + ) + assert summary["pre_tax_premium_total"] == pytest.approx(pre_tax, rel=1e-3) + assert receipt["nhe_concept_check"]["raw_all_policyholders_over_nhe"] == ( + pytest.approx(nhe_ratio, abs=5e-4) + ) + restored = ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2"] + for source in receipt["pool_census_person_columns"]: + assert source["columns_added"][-6:] == [*restored, "PEMLR", "NOEMP"] + + +# --- the cell-table extraction tool --------------------------------------------------- + + +def _load_cells_tool(): + path = _REPOSITORY_ROOT / "tools" / "build_us_meps_ic_esi_cells.py" + spec = importlib.util.spec_from_file_location("build_us_meps_ic_esi_cells", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class TestCellsTool: + def test_committed_cells_carry_the_tools_pins(self) -> None: + tool = _load_cells_tool() + assert {pin.key: pin.sha256 for pin in tool.PDFS} == { + key: source["sha256"] for key, source in _CELLS["sources"].items() + } + assert tool.render(dict(_CELLS)) == tool.OUTPUT_PATH.read_text() + + def test_row_parser_reads_suppression_and_reliability_marks(self) -> None: + tool = _load_cells_tool() + block = [ + "Table II.D.2 Average total employee contribution: United States, 2025", + "United States 7,314 6,113 7,721 9,242 7,310 7,050 7,760 7,246", + "Alaska 6,993 -- 5,811 * 8,001 7,406 6,750 -- 7,184", + "West Virginia 6,100 5,000 5,500 6,000 6,300 6,200 5,400 6,250", + "Virginia 7,000 6,500 6,800 7,100 7,200 6,900 6,700 7,050", + ] + values, unreliable = tool._row_values(block, "Alaska", 8, "II.D.2") + assert values == [6993, None, 5811, 8001, 7406, 6750, None, 7184] + assert unreliable == [False, False, True, False, False, False, False, False] + virginia, _ = tool._row_values(block, "Virginia", 8, "II.D.2") + assert virginia[0] == 7000 + + def test_row_parser_refuses_a_missing_duplicated_or_short_row(self) -> None: + tool = _load_cells_tool() + row = "Ohio 1 2 3" + with pytest.raises(tool.ExtractionError, match="matched 0 rows"): + tool._row_values([row], "Iowa", 3, "X") + with pytest.raises(tool.ExtractionError, match="matched 2 rows"): + tool._row_values([row, row], "Ohio", 3, "X") + with pytest.raises(tool.ExtractionError, match="has 3 values, not 8"): + tool._row_values([row], "Ohio", 8, "X") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 0c8c7c284e..919d926661 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -6403,11 +6403,26 @@ def fake_smoke(*, simulate, period): ), } expected_gate_evidence = { + "esi_premiums": "bound", "input_coverage": "bound", "input_mass_parity": "bound", "qrf_tail_concentration": "bound", "reform_coverage_smoke": "skipped" if skipped_smoke else "bound", } + # microcosm#454: the ESI premium signal gate graded the base before the + # solve and the export after it; the anchor gate graded the export at the + # release period and its verdict is the bound esi_premiums evidence. + assert captured["esi_premiums_signal_gate_calls"] == 2 + assert captured["esi_premiums_anchor_periods"] == [builder.PERIOD] + assert json.loads((release_dir / "esi_premiums.json").read_text()) == { + "schema_version": 1, + "enforced": True, + "esi_premiums": { + "passed": True, + "failures": [], + "details": {"checked": True}, + }, + } for block in (build_manifest, release_manifest["build"]): assert block["gate_evidence"] == expected_gate_evidence assert block["qrf_tail_register"] == expected_register @@ -8253,6 +8268,32 @@ def fake_other_health_insurance_signal_gate(frame): "us_other_health_insurance_signal_gate", fake_other_health_insurance_signal_gate, ) + + def fake_esi_premiums_signal_gate(frame): + # Called on the base before the solve and on the export after it. + captured["esi_premiums_signal_gate_calls"] = ( + captured.get("esi_premiums_signal_gate_calls", 0) + 1 + ) + return builder.GateResult( + name="esi_premiums_signal", + passed=True, + details={"checked": True}, + ) + + def fake_esi_premiums_anchor_gate(frame, *, time_period): + captured.setdefault("esi_premiums_anchor_periods", []).append(time_period) + return builder.GateResult( + name="esi_premiums_anchor", + passed=True, + details={"checked": True}, + ) + + monkeypatch.setattr( + builder, "us_esi_premiums_signal_gate", fake_esi_premiums_signal_gate + ) + monkeypatch.setattr( + builder, "us_esi_premiums_anchor_gate", fake_esi_premiums_anchor_gate + ) monkeypatch.setattr( builder, "_with_snap_state_take_up_outputs", @@ -12357,7 +12398,7 @@ def test_build_manifests_binds_gate_evidence_and_the_qrf_tail_register( monkeypatch, tmp_path ) -> None: """Route A remediation PR-3: a certified waiver must ship with the - release. The four gate verdicts become release artifacts, and both + release. The five gate verdicts become release artifacts, and both manifests record the register the tail gate evaluated, the export-mass reference, the solve's thread geometry and the exclusion receipt.""" from microcosm.data.contract import ( @@ -12394,7 +12435,11 @@ def test_build_manifests_binds_gate_evidence_and_the_qrf_tail_register( == [] ) assert failures == [] - for filename in ("input_coverage.json", "input_mass_parity.json"): + for filename in ( + "esi_premiums.json", + "input_coverage.json", + "input_mass_parity.json", + ): (release_dir / filename).write_text('{"schema_version": 1, "enforced": true}') (release_dir / "reform_coverage_smoke.json").write_text( '{"schema_version": 1, "enforced": true}' @@ -12520,6 +12565,7 @@ def test_build_manifests_records_a_blanket_waiver_without_register_and_absent_re assert build["fiscal_target_exclusion_receipt"]["receipt_sha256"] is None # A verdict that was not written is named, never silently missing. assert build["gate_evidence"] == { + "esi_premiums": "not_evaluated", "input_coverage": "not_evaluated", "input_mass_parity": "not_evaluated", "qrf_tail_concentration": "bound", @@ -12544,13 +12590,14 @@ def test_build_manifests_without_gate_evidence_binds_none() -> None: def test_gate_evidence_status_names_skipped_and_unevaluated_gates() -> None: """A skipped smoke and a gate that crashed under earlier failures both leave no verdict; the status block tells them apart. A gate cannot be - both skipped and bound, and only the four bound gates can be skipped.""" + both skipped and bound, and only the five bound gates can be skipped.""" builder = _load_builder_module() bound = {"input_coverage": {}, "input_mass_parity": {}} assert builder._gate_evidence_status( bound, skipped_gates=("reform_coverage_smoke",) ) == { + "esi_premiums": "not_evaluated", "input_coverage": "bound", "input_mass_parity": "bound", "qrf_tail_concentration": "not_evaluated", diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py index bd3d3c2284..2a65e9476a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py @@ -561,77 +561,43 @@ def test_legacy_two_column_surface_is_rebuilt_or_refused( ) -def test_esi_exclusion_pins_complete_hermetic_source_unavailability_evidence() -> None: +def test_esi_employer_premium_is_owned_by_its_own_stage_not_proxied_here() -> None: + """microcosm#454 closed the source-unavailability gap this test once pinned. + + The retired derivation's NOW_* fields are restored from the pinned Census + person files and the ``meps_esi_premiums`` stage reads them. This stage + still never proxies the employer premium, and neither stage may fall back + on ``has_esi`` or ``NOW_GRP``. + """ + payload = json.loads( files("microcosm.build.us").joinpath("ecps_parity_known_gaps.json").read_text() ) - entry = payload["known_gaps"]["employer_sponsored_insurance_premiums"] - evidence = entry["evidence"] - - assert entry["reason"].startswith("SOURCE UNAVAILABILITY WITH EVIDENCE:") - assert evidence["classification"] == "source_unavailability" - assert evidence["retired_derivation"] == { - "repository_owner": "PolicyEngine", - "repository_name_parts": ["policyengine-", "us-data"], - "commit": "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe", - "path_parts": ["policyengine_", "us_data", "datasets", "cps", "cps.py"], - "lines": "197-271,1575-1581", - } - assert evidence["optional_source_fields"]["lines"] == "13-55" - assert evidence["required_columns"] == [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - "PHIP_VAL", - ] - evidence_hashes = { - item["filename"]: item["sha256"] for item in evidence["hermetic_inputs"] - } - build_summary = json.loads( - (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() - ) - recorded_hashes = { - Path(item["path"]).name: item["sha256"] - for item in build_summary["base_source"]["sources"] - } - assert evidence_hashes == recorded_hashes - assert set(evidence_hashes) == { - "census_cps_2022.h5", - "census_cps_2023.h5", - "census_cps_2024.h5", - } - for item in evidence["hermetic_inputs"]: - assert item["present_columns"] == ["PHIP_VAL"] - assert item["missing_columns"] == [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - ] - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] - build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() - for year in (2022, 2023, 2024): - assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script + assert "employer_sponsored_insurance_premiums" not in payload["known_gaps"] esi_stage = US_SOURCE_MANIFEST.stage_map()["meps_esi_premiums"] - assignment = next( - operation - for operation in esi_stage.operations - if operation.kind == "assign_by_plan_type" + assert "employer_sponsored_insurance_premiums" in esi_stage.outputs + assert "employer_sponsored_insurance_premiums" not in ( + us_other_health_insurance_stage_spec().outputs + ) + microdata = next( + artifact + for artifact in esi_stage.artifacts + if artifact["kind"] == "public_microdata" ) - assert assignment.parameters["inputs"] == [ + source_columns = set(microdata["source_columns"]) + assert { "NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", + "NOW_GRPFTYP2", "PHIP_VAL", - ] - assert assignment.parameters["source_unavailable_when_missing"] == [ - "NOW_OWNGRP", - "NOW_HIPAID", - "NOW_GRPFTYP", - ] - forbidden_proxies = {"has_esi", "is_esi_dependent", "tax_unit_size", "state_fips"} - assert forbidden_proxies.isdisjoint(assignment.parameters["inputs"]) + } <= source_columns + forbidden_proxies = {"has_esi", "NOW_GRP", "is_esi_dependent", "tax_unit_size"} + assert forbidden_proxies.isdisjoint(source_columns) + assert not any( + operation.kind == "assign_by_plan_type" for operation in esi_stage.operations + ) def test_shipped_se_health_premium_neutralization_probe() -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py index 6bbccdc296..51fa1a802f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_plan.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_plan.py @@ -1275,6 +1275,10 @@ class TestBaseStageSourceClosure: "source_person_id", "source_row_id", "person_source_id", + # The household State (GESTFIPS) the pool attaches to every person + # and unit construction moves to the household table; the + # meps_esi_premiums stage joins it back by person_household_id. + "state_fips", } ) @@ -1283,6 +1287,14 @@ class TestBaseStageSourceClosure: #: 2022; ED_VAL and PAW_TYP for every pooled year). SIDECAR_RESTORED_COLUMNS = frozenset({"LKWEEKS", "ED_VAL", "PAW_TYP"}) + #: Census person columns asec_census_person_columns restores into every + #: pooled vintage, by exact PERIDNUM identity, that NO frozen input carries + #: (microcosm #454): the meps_esi_premiums stage's coverage, employment + #: and employer-size fields. + CENSUS_PERSON_RESTORED_COLUMNS = frozenset( + {"NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2", "PEMLR", "NOEMP"} + ) + #: Release-time stage constants whose inputs are produced inside the #: fiscal-refresh release tool, not the base builder (org_wages consumes #: hours/weeks columns the release derives upstream of it). @@ -1323,6 +1335,7 @@ def test_every_base_stage_requirement_has_a_provider(self) -> None: self.CENSUS_CPS_PERSON_COLUMNS | self.POOL_CONSTRUCTED_COLUMNS | self.SIDECAR_RESTORED_COLUMNS + | self.CENSUS_PERSON_RESTORED_COLUMNS | produced ) unsourced: dict[str, list[str]] = {} @@ -1338,6 +1351,15 @@ def test_every_base_stage_requirement_has_a_provider(self) -> None: ] if missing: unsourced[name] = missing + from microcosm.build.us_runtime.asec_census_person_columns import ( + ASEC_CENSUS_PERSON_COLUMN_NAMES, + ) + + # The restoration really does carry every column declared restored. + assert self.CENSUS_PERSON_RESTORED_COLUMNS <= set( + ASEC_CENSUS_PERSON_COLUMN_NAMES + ) + assert not self.CENSUS_PERSON_RESTORED_COLUMNS & self.CENSUS_CPS_PERSON_COLUMNS assert unsourced == {}, ( "Stage source columns with no declared provider (wire an ingestion " f"carry, a pinned sidecar, or an upstream stage output): {unsourced}" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py index c5208021c7..c603c42d33 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py @@ -75,7 +75,7 @@ def test_batched_engines_partition_release_and_order_their_work( validation = consumers["reform_validation"].record() demographics = consumers["demographics"].record() probes = len(us_release_reform_coverage_probes()) - assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 41 + assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 43 assert validation["reform_systems"] == validation["reform_passes"] > 0 assert demographics["reform_systems"] == demographics["reform_passes"] == 0 assert all(engine.reform is None for engine in constructions) @@ -1377,7 +1377,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( ) -> None: sweep = _load_sweep_module() plans, requests, counts = sweep._record_requests(builder) - assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 41 + assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 43 specs = reform_validation_module.load_default_reform_specs(period=builder.PERIOD) assert counts["validation_specs"] == sum(not spec.in_sample for spec in specs) == 52 assert ( @@ -1386,7 +1386,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( == 12 ) assert counts["baseline_keys"] == { - "reform_coverage_smoke": 18, + "reform_coverage_smoke": 19, "reform_validation": 75, "demographics": 1, } @@ -1421,7 +1421,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( assert ( len(requests) == counts["smoke_probes"] + counts["validation_reform_passes"] - == 107 + == 109 ) frame, log = _nested_frame(), _EngineLog() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py index e6944adaae..82ebe5574e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_puf_support_base_builder.py @@ -1000,6 +1000,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", + "with_us_esi_premium_inputs", "with_us_immigration_inputs", ) for name in identity_transforms: @@ -1013,6 +1014,7 @@ def add_age(frame: Frame, **_kwargs) -> Frame: "us_eligibility_inputs_signal_gate", "us_pregnancy_signal_gate", "us_wic_claim_signal_gate", + "us_esi_premiums_signal_gate", ): monkeypatch.setattr(builder, name, lambda _frame: passing_gate) @@ -1180,6 +1182,7 @@ def test_source_and_preclone_stages_round_trip_design_weight_kind( "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", + "with_us_esi_premium_inputs", "with_us_immigration_inputs", ) for name in identity_transforms: @@ -1193,6 +1196,7 @@ def test_source_and_preclone_stages_round_trip_design_weight_kind( "us_eligibility_inputs_signal_gate", "us_pregnancy_signal_gate", "us_wic_claim_signal_gate", + "us_esi_premiums_signal_gate", ): monkeypatch.setattr(builder, name, lambda _frame: passing_gate) monkeypatch.setattr(builder, "_sha256", lambda _path: "fixture-sha256") @@ -2322,6 +2326,7 @@ def fake_housing_inputs_signal_gate(frame): for gate_name in ( "us_eligibility_inputs_signal_gate", "us_pregnancy_signal_gate", + "us_esi_premiums_signal_gate", ): monkeypatch.setattr( builder, @@ -2330,6 +2335,13 @@ def fake_housing_inputs_signal_gate(frame): "Gate", (), {"passed": True, "failures": (), "details": {}} )(), ) + esi_premium_calls: list[object] = [] + + def fake_esi_premium_inputs(frame, *, seed, time_period): + esi_premium_calls.append((frame, seed, time_period)) + return frame + + monkeypatch.setattr(builder, "with_us_esi_premium_inputs", fake_esi_premium_inputs) monkeypatch.setattr( builder, "us_wic_claim_signal_gate", @@ -2862,6 +2874,10 @@ def fake_retirement_distributions_signal_gate(frame): if failing_gate == "retirement_distributions" else [] ) + # microcosm#454: the ESI premium stage runs once, on the ASEC frame right + # after the direct retirement-distribution pass and before the clone; the + # clone carries its columns unchanged, so it never re-runs on the PUF half. + assert esi_premium_calls == [("retirement-distributions-direct", 7, 2024)] def test_resume_retirement_stage_forces_puf_imputation() -> None: @@ -2890,6 +2906,29 @@ def test_main_summary_records_retirement_distribution_gate() -> None: assert '"details": dict(retirement_distributions_gate.details)' in source +def test_main_summary_records_esi_premium_gates() -> None: + """microcosm#454: the base summary carries the stage lineage twice, on the + ASEC frame and again on the support-cloned frame, in both build paths.""" + builder = _load_support_builder_module() + source = Path(builder.__file__).read_text(encoding="utf-8") + + assert '"esi_premiums_signal": {' in source + assert '"passed": esi_premiums_gate.passed' in source + assert '"details": dict(esi_premiums_gate.details)' in source + assert '"esi_premiums_post_clone_signal": {' in source + assert '"details": dict(esi_premiums_post_clone_gate.details)' in source + assert 'signals["esi_premiums_signal"] = _checked_gate_payload(' in source + assert ( + 'signals["esi_premiums_post_clone_signal"] = _checked_gate_payload(' in source + ) + for name in ("esi_premiums_signal", "esi_premiums_post_clone_signal"): + assert f' "{name}",' in source # the staged summary requires it + boundaries = dict(builder.STAGE_BOUNDARIES)["pre_clone_enrichment"] + assert boundaries.index("with_us_esi_premium_inputs") == ( + boundaries.index("with_us_retirement_distribution_inputs") + 1 + ) + + def test_main_summary_records_weeks_unemployed_gate() -> None: builder = _load_support_builder_module() source = Path(builder.__file__).read_text(encoding="utf-8") diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py index 789b149a05..896f3cf3b9 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py @@ -258,7 +258,7 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( "identity_generation": 1, "seed_protocol": LEGACY_V1_PROTOCOL.id, } - assert len(LEGACY_V1_PROTOCOL.sites) == 53 + assert len(LEGACY_V1_PROTOCOL.sites) == 54 assert len(LEGACY_V1_PROTOCOL.streams) == 14 assert LEGACY_V1_PROTOCOL.site("survey_sample_asec").default == 578 assert LEGACY_V1_PROTOCOL.site("puf_live_aggregate_disaggregation").default == 0 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 1449c2e277..777cf5b5eb 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -266,6 +266,11 @@ "education_assistance_source.py", "eligibility_inputs.py", "engine_lifecycle.py", + # MEPS-IC employer-premium and pre-tax premium inputs (#454): a + # base-builder pre-clone stage keyed on restored ASEC coverage fields, + # not yet a multispine pool operator. Reads no spine or support-channel + # column and remains subject to the all-runtime source-identity scan. + "esi_premiums.py", "exact_k_ladder.py", # Calibration/selection seam; no source treatment. "fiscal_targets.py", "geography_ladder.py", diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 9606d75b4c..3c150fc2d6 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "f76f921d4f74392b106b8c0729ce250494e0a55455eaba7b910b19cdf7825178", + "spec_sha256": "e9360b25997322886028b7a8a8964007201e2b36b7bbd341d0b483093e026950", }, } diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py index 45f7c2c958..08c7ab2714 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_post_export_scoring.py @@ -277,7 +277,7 @@ def test_shipped_baseline_plans_match_whole_file_policyengine_us( path = _write_engine_h5(builder, tmp_path, _GUARD_HOUSEHOLDS) plans, _, counts = sweep._record_requests(builder) assert counts["baseline_keys"] == { - "reform_coverage_smoke": 18, + "reform_coverage_smoke": 19, "reform_validation": 75, "demographics": 1, } diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 1d051f5c4d..26072ef5e6 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -124,9 +124,9 @@ def binding() -> dict[str, object]: "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 112, - "seed_owner_rows": 54, - "seed_sites": 53, + "seed_owner_bindings": 113, + "seed_owner_rows": 55, + "seed_sites": 54, "seed_streams": 14, "source_operators": 16, "source_stages": 38, diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index ef4bc8868b..b03c7c8050 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 4d7e1dd955..f8b7b21f53 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -136,6 +136,8 @@ us_educator_expense_signal_gate, us_eligibility_inputs_signal_gate, us_energy_subsidy_signal_gate, + us_esi_premiums_anchor_gate, + us_esi_premiums_signal_gate, us_farm_business_income_signal_gate, us_fiscal_target_exclusion_receipt, us_form_4952_election_signal_gate, @@ -1301,6 +1303,18 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: "certification blocker." ), ) + parser.add_argument( + "--allow-esi-premium-gaps", + action="store_true", + help=( + "Diagnostic escape hatch (microcosm#454): record the ESI premium " + "gates — employer_sponsored_insurance_premiums and " + "pre_tax_health_insurance_premiums absent, zero-mass, or off the " + "BEA NIPA 7.8 anchor — without failing the build (for example on " + "a base built before the meps_esi_premiums stage). Release builds " + "must leave this unset." + ), + ) parser.add_argument( "--allow-qrf-tail-concentration", action="store_true", @@ -9945,6 +9959,7 @@ def _artifact_entry(path: str, sha: str, *, kind: str, revision: str) -> dict[st #: before the terminal gates run, so any present at manifest time are this #: run's own. US_RELEASE_GATE_EVIDENCE_FILES: dict[str, str] = { + "esi_premiums": "esi_premiums.json", "input_coverage": "input_coverage.json", "input_mass_parity": "input_mass_parity.json", "qrf_tail_concentration": "qrf_tail_concentration.json", @@ -11499,6 +11514,7 @@ def inputs(self) -> dict[str, object]: "allow_input_mass_drift": bool(args.allow_input_mass_drift), "allow_ecps_parity_gaps": bool(args.allow_ecps_parity_gaps), "allow_input_coverage_gaps": bool(args.allow_input_coverage_gaps), + "allow_esi_premium_gaps": bool(args.allow_esi_premium_gaps), }, "margins": self.margins.to_dict(), # Taken when _main reached the stop point, before any grading; @@ -14117,6 +14133,25 @@ def _main(argv: Sequence[str] | None = None) -> int | None: for failure in other_health_insurance_gate.failures ) ) + # microcosm#454: the meps_esi_premiums base stage owns both ESI premium + # inputs; refuse a base that lacks or flattens them before the solve. + esi_premiums_base_gate = us_esi_premiums_signal_gate(base_frame) + if not esi_premiums_base_gate.passed and not args.allow_esi_premium_gaps: + if telemetry is not None: + telemetry.stage( + "esi_premiums_input_gate", + status="failed", + message="ESI premium signal gate failed on the base.", + failures=list(esi_premiums_base_gate.failures), + force_upload=True, + ) + raise RuntimeError( + "Release gates failed: " + + "; ".join( + "ESI premium signal failed: " + failure + for failure in esi_premiums_base_gate.failures + ) + ) if telemetry is not None and args.input_mass_reference_h5 is not None: telemetry.stage( "input_mass_reference_gate", @@ -14879,6 +14914,25 @@ def _main(argv: Sequence[str] | None = None) -> int | None: f"Other health insurance signal failed on the export frame: {failure}" for failure in other_health_insurance_gate.failures ) + try: + esi_premiums_export_gate = us_esi_premiums_signal_gate(export_frame) + except Exception as error: + if not early_terminal_gate_failures: + raise + esi_premiums_export_gate = None + early_terminal_gate_failures.append( + "ESI premium signal evaluation crashed in degraded mode; recorded " + f"instead of masking earlier failures: {error}" + ) + if ( + esi_premiums_export_gate is not None + and not esi_premiums_export_gate.passed + and not args.allow_esi_premium_gaps + ): + early_terminal_gate_failures.extend( + f"ESI premium signal failed on the export frame: {failure}" + for failure in esi_premiums_export_gate.failures + ) if congressional_district_vintage_crosswalk_metadata is not None: compilation = { **compilation, @@ -15296,6 +15350,58 @@ def _main(argv: Sequence[str] | None = None) -> int | None: failures=list(input_coverage_gate.failures), force_upload=True, ) + # microcosm#454: the calibrated ESI premium totals against their external + # anchors (BEA NIPA 7.8 line 17 within tolerance; at most the + # retiree-inclusive NHE Table 24), red when either column is absent or + # zero-mass. Runs on the calibrated export for BOTH the dense and sparse + # default paths; the verdict ships as the esi_premiums gate evidence. + try: + esi_premiums_anchor_gate = us_esi_premiums_anchor_gate( + export_frame, time_period=PERIOD + ) + except Exception as exc: + if not terminal_gate_failures: + raise + terminal_gate_failures.append( + "ESI premiums failed: evaluation error under earlier gate " + f"failures: {type(exc).__name__}: {exc}" + ) + esi_premiums_anchor_gate = None + if esi_premiums_anchor_gate is not None: + esi_premiums_failed = ( + not esi_premiums_anchor_gate.passed and not args.allow_esi_premium_gaps + ) + if esi_premiums_failed: + terminal_gate_failures.extend( + f"ESI premiums failed: {failure}" + for failure in esi_premiums_anchor_gate.failures + ) + esi_premiums_path = release_dir / "esi_premiums.json" + esi_premiums_path.write_text( + json.dumps( + { + "schema_version": 1, + "enforced": not args.allow_esi_premium_gaps, + "esi_premiums": { + "passed": esi_premiums_anchor_gate.passed, + "failures": list(esi_premiums_anchor_gate.failures), + "details": dict(esi_premiums_anchor_gate.details), + }, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + terminal_batch_telemetry.attach_artifact("esi_premiums", esi_premiums_path) + if esi_premiums_failed: + terminal_batch_telemetry.stage( + "export_dataset", + status="failed", + message="ESI premium anchor gate failed.", + failures=list(esi_premiums_anchor_gate.failures), + force_upload=True, + ) # #327: the export gate compares the calibrated export against a reference. # By default that reference is the raw pre-calibration base — but for a dense # parent built from a raw pooled-ASEC base, calibration correctly scales diff --git a/tools/build_us_puf_support_base.py b/tools/build_us_puf_support_base.py index b5f8663ea1..1ec4cafa1b 100644 --- a/tools/build_us_puf_support_base.py +++ b/tools/build_us_puf_support_base.py @@ -105,6 +105,7 @@ us_educator_expense_signal_gate, us_eligibility_inputs_signal_gate, us_energy_subsidy_signal_gate, + us_esi_premiums_signal_gate, us_farm_business_income_signal_gate, us_form_4952_election_signal_gate, us_geography_ladder_assignment_summary, @@ -136,6 +137,7 @@ with_us_education_inputs, with_us_eligibility_inputs, with_us_energy_subsidy_input, + with_us_esi_premium_inputs, with_us_housing_inputs, with_us_immigration_inputs, with_us_medicare_take_up_input, @@ -224,6 +226,7 @@ "with_us_energy_subsidy_input", "with_us_retirement_contribution_inputs", "with_us_retirement_distribution_inputs", + "with_us_esi_premium_inputs", "with_us_immigration_inputs", ), ), @@ -1217,6 +1220,17 @@ def _run_all( seed=args.seed, time_period=args.target_year, ) + base = with_us_esi_premium_inputs( + base, + seed=args.seed, + time_period=args.target_year, + ) + esi_premiums_gate = us_esi_premiums_signal_gate(base) + if not esi_premiums_gate.passed: + raise SystemExit( + "ESI premium signal gate failed before support cloning:\n " + + "\n ".join(esi_premiums_gate.failures) + ) base = with_us_immigration_inputs( base, seed=args.seed, @@ -1515,6 +1529,14 @@ def _run_all( "Education-input signal gate failed:\n " + "\n ".join(education_inputs_gate.failures) ) + # The ESI premium inputs ride the support clone unchanged; re-prove clone + # agreement and the cell multiples on the cloned frame. + esi_premiums_post_clone_gate = us_esi_premiums_signal_gate(imputed) + if not esi_premiums_post_clone_gate.passed: + raise SystemExit( + "ESI premium signal gate failed after support cloning:\n " + + "\n ".join(esi_premiums_post_clone_gate.failures) + ) _observe_frame_boundary(boundary_observer, "education_inputs_post_clone", imputed) congressional_district_assignment = {"applied": False} if args.assign_congressional_districts: @@ -1725,6 +1747,16 @@ def _run_all( "failures": list(pregnancy_gate.failures), "details": dict(pregnancy_gate.details), }, + "esi_premiums_signal": { + "passed": esi_premiums_gate.passed, + "failures": list(esi_premiums_gate.failures), + "details": dict(esi_premiums_gate.details), + }, + "esi_premiums_post_clone_signal": { + "passed": esi_premiums_post_clone_gate.passed, + "failures": list(esi_premiums_post_clone_gate.failures), + "details": dict(esi_premiums_post_clone_gate.details), + }, "wic_claim_signal": { "passed": wic_claim_gate.passed, "failures": list(wic_claim_gate.failures), @@ -2328,6 +2360,15 @@ def _pre_clone_enrichment_stage( seed=args.seed, time_period=args.target_year, ) + base = with_us_esi_premium_inputs( + base, + seed=args.seed, + time_period=args.target_year, + ) + signals["esi_premiums_signal"] = _checked_gate_payload( + us_esi_premiums_signal_gate(base), + "ESI premium signal gate failed before support cloning", + ) base = with_us_immigration_inputs( base, seed=args.seed, @@ -2874,6 +2915,10 @@ def _post_qrf_frame_stage( us_education_inputs_signal_gate(frame), "Education-input signal gate failed", ) + signals["esi_premiums_post_clone_signal"] = _checked_gate_payload( + us_esi_premiums_signal_gate(frame), + "ESI premium signal gate failed after support cloning", + ) elif stage == "congressional_district_assignment": assignment: dict[str, object] = {"applied": False} if args.assign_congressional_districts: @@ -3021,6 +3066,8 @@ def _export_staged_result( "weeks_unemployed_signal", "eligibility_inputs_signal", "pregnancy_signal", + "esi_premiums_signal", + "esi_premiums_post_clone_signal", "wic_claim_signal", "educator_expense_signal", "form_4952_election_signal", diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index be6f9f56f2..12c8e968a1 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -89,6 +89,9 @@ "pre_subsidy_care_expenses", "is_incapable_of_self_care", "is_spm_independent_minor_role", + # PolicyEngine/microcosm#454: the employee section 125 premium leaf the + # meps_esi_premiums base stage writes (the reference eCPS never had one). + "pre_tax_health_insurance_premiums", # PolicyEngine/microcosm#978 option 1: the ASEC reported-receipt inputs the # ACS local-area transfer requires in its donor. "receives_wic", @@ -113,6 +116,30 @@ # from today's artifacts is the intended red gate (the #368 SSI-asset # pattern, extended by #462 to the capital-gain-distributions route). POST_REFERENCE_COLUMN_NOTES = { + "employer_sponsored_insurance_premiums": ( + "Employer share of the premium for employed current ESI policyholders " + "(PolicyEngine/microcosm#454), written by the meps_esi_premiums " + "base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / " + "NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person " + "files) and MEPS-IC tier x firm-size x State cells, scaled to BEA NIPA " + "7.8 line 17. A release must persist it: CBO household market income " + "(gov.household.cbo_market_income_additions) and every ESI-to-wages " + "reform bind through it. Required with NO reviewed exclusion; absent " + "on pre-#454 bases by construction — the intended red gate until a " + "base built with the stage ships. The release also holds its total to " + "the anchor (us_esi_premiums_anchor_gate)." + ), + "pre_tax_health_insurance_premiums": ( + "Employee premium paid through section 125 payroll deduction " + "(PolicyEngine/microcosm#454), written by the same meps_esi_premiums " + "stage: an employed policyholder with wages, a reported premium " + "(PHIP_VAL) and an employee share pays it pre-tax at the MEPS-IC 2025 " + "firm-size pretax-offer rate. PolicyEngine-US 2.2.1 subtracts it from " + "income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, " + "fica_pre_tax_contributions); absent, both wage bases overstate. " + "Required with NO reviewed exclusion; absent on pre-#454 bases by " + "construction." + ), "schedule_d_capital_gain_distributions": ( "Schedule D line 13 route leg of the #282 capital-gain-distributions " "split (memo component of long_term_capital_gains, written by the " @@ -229,6 +256,8 @@ OTHER_HEALTH_INSURANCE_INPUTS = ("other_health_insurance_premiums",) +ESI_PREMIUM_INPUTS = ("employer_sponsored_insurance_premiums",) + PRIOR_YEAR_INCOME_INPUTS = ( "self_employment_income_last_year", "previous_year_income_available", @@ -300,6 +329,7 @@ *WIC_CLAIM_INPUTS, *EDUCATOR_EXPENSE_INPUTS, *OTHER_HEALTH_INSURANCE_INPUTS, + *ESI_PREMIUM_INPUTS, *FARM_BUSINESS_INCOME_INPUTS, *SCF_NET_WORTH_INPUTS, *SIPP_VEHICLE_INPUTS, @@ -347,6 +377,55 @@ #: a floor far below the plausible effect but far above simulation noise, so a #: structural $0 fails while a real (even conservative) score passes. REFORM_COVERAGE_PROBES = [ + { + "id": "employer_sponsored_insurance_premium_neutralization", + "name": "Employer-sponsored insurance premium neutralization", + "parameter_changes": {}, + "neutralized_variable": "employer_sponsored_insurance_premiums", + "budget_measure": "cbo_household_market_income", + "period": 2024, + "effect_direction": "baseline_minus_reform", + "expected_sign": "positive", + "binding_inputs": ["employer_sponsored_insurance_premiums"], + "min_abs_effect": 500_000_000_000.0, + "reason": ( + "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums " + "to CBO household market income through " + "gov.household.cbo_market_income_additions, its only consumer, so " + "neutralizing it lowers cbo_household_market_income by the " + "weighted employer-premium total: about $977 billion at 2024 " + "(BEA NIPA 7.8 line 17, the meps_esi_premiums anchor; the stage " + "measured $977.0 billion over 74.0 million weighted employed " + "policyholders on the default 2023-2025 pool, before calibration). " + "The floor sits at about half of that. A structural zero means " + "the column was dropped or the stage never ran." + ), + "issue": "PolicyEngine/microcosm#454", + }, + { + "id": "pre_tax_health_insurance_premium_neutralization", + "name": "Pre-tax health insurance premium neutralization", + "parameter_changes": {}, + "neutralized_variable": "pre_tax_health_insurance_premiums", + "budget_measure": "income_tax", + "period": 2024, + "effect_direction": "baseline_minus_reform", + "expected_sign": "negative", + "binding_inputs": ["pre_tax_health_insurance_premiums"], + "min_abs_effect": 5_000_000_000.0, + "reason": ( + "PolicyEngine-US 2.2.1 subtracts pre_tax_health_insurance_premiums " + "from employment_income in irs_employment_income (through " + "gov.irs.gross_income.pre_tax_contributions), so neutralizing it " + "raises taxable wages and income tax: baseline-minus-reform is " + "negative. The meps_esi_premiums stage measured $181.1 billion of " + "pre-tax premiums over 52.5 million weighted workers on the default " + "2023-2025 pool; at a typical 10-22 percent marginal rate that is " + "roughly $18-40 billion, so a $5 billion floor separates a " + "structural zero (the column dropped) from any plausible effect." + ), + "issue": "PolicyEngine/microcosm#454", + }, { "id": "prior_year_self_employment_neutralization", "name": "Prior-year self-employment support neutralization", @@ -1627,7 +1706,9 @@ def build_manifest() -> dict: "(PolicyEngine/microcosm#978 option 1), and the engine's declared " "dataset source input is_spm_independent_minor_role (the " "spm_independence_role base-builder stage; " - "PolicyEngine/microcosm#893). " + "PolicyEngine/microcosm#893), and the employee section 125 " + "premium leaf pre_tax_health_insurance_premiums (the " + "meps_esi_premiums base-builder stage; PolicyEngine/microcosm#454). " "status='reviewed_exclusion' for ecps_parity_known_gaps.json entries " "(reason+issue from that register); EXCEPT every primary-source " "restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS " diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 57d3238ba9..c5040ef43a 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -240,19 +240,24 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: ) _ESI_ANCHOR_EXCLUSION = ( - "not_modeled", - "NHE employer-contribution ESI premium aggregate: a retired us-data " - "calibration target (loss.py 1_002.9e9, removed at 9f320d2c) whose " - "anchor v9.2 re-banks (2026-07-21) for microcosm#454's future gate. The pe-us input it would anchor " - "(employer_sponsored_insurance_premiums, the employer-paid share per " - "the variable's own documentation) has no live producer in the " - "hermetic lineage, so a target would bind against a structural zero. " - "The all-employer series is the future anchor (pe-us covers government " - "and private employees alike); the private-employer subset rides as its " - "cross-check.", - "ecps_parity_known_gaps.json entry employer_sponsored_insurance_" - "premiums (NOW_* source-unavailability evidence); microcosm#454 " - "descope comment 2026-07-21", + "deferred", + "NHE Table 24 employer contribution to ESI premiums. The pe-us input it " + "anchors (employer_sponsored_insurance_premiums) is now produced by the " + "meps_esi_premiums base stage (microcosm#454) for employed current ESI " + "policyholders only, while NHE counts COBRA and retiree enrollees (NHEA " + "methodology paper 2024, p. 28; Table 24 note 1 adds Medicare Retiree " + "Drug Subsidy payments), so a calibration target on this fact would load " + "the retiree share onto workers. Validation-gated instead: " + "us_esi_premiums_anchor_gate holds the calibrated total within 5% of BEA " + "NIPA 7.8 line 17 (the us-data target's concept, pinned in the stage) and " + "at or below this all-employer fact; the private-employer subset rides as " + "the sector cross-check.", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: MEPS-IC " + "cells over all 92.4M weighted current policyholders give $1,031.0B " + "(0.985 x NHE $1,047.0B CY2024); the employed universe gives $912.8B raw, " + "scaled to BEA $977.0B (11.5% of the all-policyholder total is " + "non-employed or no-employer coverage); private employers $756.7B vs NHE " + "private $753.4B", _fence( origin=( "a REAL us-data calibration target: utils/loss.py pinned " @@ -263,27 +268,25 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "national calibration targets', PR #831). The input itself was " "imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* " "policyholder fields, archived 42ed5d45 cps.py " - "L197-271/L1575-1581). The v9.2 feed facts re-bank that same " - "anchor for the future gate, entered deliberately ahead of the " - "producer." + "L197-271/L1575-1581). The v9.2 feed banks the NHE series for the " + "#454 gate; the NIPA series the us-data target used is not in the " + "feed (source-absent family bea_nipa.private_group_health_insurance)." ), purpose=( - "anchor a validation gate for the CBO market-income ESI " - "component once the column has a live producer: NIPA 7.8 " - "employer contributions for group health insurance and the " - "NHEA sponsor-of-funds employer share are same-concept " - "estimates (~4 percent apart, CY2024)." + "anchor the CBO market-income ESI component once the column has a " + "live producer. NIPA 7.8 line 17 counts employer contributions as " + "compensation of current employees (BEA builds it from MEPS-IC " + "'insurance purchased by employers for employees'); NHE Table 24 " + "adds COBRA and retiree enrollees, so it bounds the column from " + "above (7.2% above BEA in CY2024)." ), verdict_basis=( - "us-data-targeted, so under the absolute rule this compiles " - "UNLESS source-absent — and the producer source is absent: the " - "declared meps_esi_premiums stage cannot execute because all " - "three required NOW_* fields are missing from the sha-pinned " - "2022-2024 ASEC h5 inputs (recorded parity known-gap), so a " - "target would bind against a structural zero. Compiles when " - "microcosm#454 restores the source fields or lands a validated " - "PHIP_VAL+MEPS re-derivation; the anchor value is already " - "banked in the feed." + "deferred, not dropped: the producer is live, but the only banked " + "fact measures a broader concept than the column. Release " + "validation (us_esi_premiums_anchor_gate, both dense and sparse " + "exports) uses it as the upper bound today; bank BEA NIPA 7.8 line " + "17 (B4923C) in the ledger and compile that as the calibration " + "target, and this family stays the cross-check." ), ), ) @@ -544,6 +547,28 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: # us-data targets the pinned feed carries NO ledger fact for. (reason, evidence, # fence) with classification source_absent. _SOURCE_ABSENT_US_DATA_FAMILIES: dict[str, tuple[str, str, dict[str, str]]] = { + "bea_nipa.private_group_health_insurance": ( + "BEA NIPA Table 7.8 line 17 private group health insurance (B4923C), " + "the employer-contribution aggregate the retired pipeline calibrated " + "employer_sponsored_insurance_premiums to (loss.py 1_002.9e9). The " + "pinned feed carries no fact for it.", + "retired us-data pipeline (archived 2e044571) loss.py line 33; " + "feed bea_nipa families carry no group-health series", + _fence( + origin="us-data loss.py employer_sponsored_insurance_premiums target", + purpose=( + "anchor employer-paid ESI premiums for current employees: the " + "compensation concept the employed-policyholder column carries." + ), + verdict_basis=( + "source-absent from the feed: the meps_esi_premiums stage pins " + "the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., " + "published 2026-09-30: $977.034B CY2024) and scales to it, and " + "the release anchor gate validates against it; compile it here " + "once the ledger banks the series." + ), + ), + ), "bls.consumer_expenditure": ( "BLS Consumer Expenditure childcare-expense target the retired pipeline " "calibrated (loss.py BLS_CE_TOTALS childcare_expenses = $63.09B). The " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 5dc45835db..19bd607938 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e75217c0db3075bf2664492167d31ce7ee82c2c5f6a70e3b67e8bb6f4ddd7ecf" + "0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index c066d573ce..a58034dc2a 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8f655670b9794adae905ede64bdceba6395eeb8d642b6c5896d5c516f1f6a6a7" + "8afbe29a5ad75630b11e561e242d1882c94efc10648d287e80c6ab0a7f44f1bc" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index 06b2bf04d8..c0d48e811e 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -102,7 +102,7 @@ def _stages_with_operation( def build_seed_site_bindings( source_document: Mapping[str, Any], ) -> list[dict[str, object]]: - """Bind all 53 legacy-v1 sites to their concrete execution owners.""" + """Bind all 54 legacy-v1 sites to their concrete execution owners.""" source_ids = _source_stage_ids(source_document) transfer_nodes = tuple(group.name for group in CANONICAL_US_LATE_TRANSFER_GROUPS) @@ -169,6 +169,7 @@ def build_seed_site_bindings( ), "snap_take_up_assignment": source("snap_take_up"), "pregnancy_assignment": source("pregnancy"), + "pre_tax_health_premium_assignment": source("meps_esi_premiums"), "wic_claim_assignment": source("wic_claim_input"), "snap_discretionary_exemption_assignment": source( "snap_abawd_discretionary_exemption" @@ -203,9 +204,9 @@ def build_seed_site_bindings( } protocol_site_ids = tuple(site.id for site in LEGACY_V1_PROTOCOL.sites) - if len(protocol_site_ids) != 53 or set(owners) != set(protocol_site_ids): + if len(protocol_site_ids) != 54 or set(owners) != set(protocol_site_ids): raise RuntimeError( - "legacy-v1 seed owner ledger must cover exactly 53 protocol sites; " + "legacy-v1 seed owner ledger must cover exactly 54 protocol sites; " f"missing={sorted(set(protocol_site_ids) - owners.keys())!r}, " f"extra={sorted(owners.keys() - set(protocol_site_ids))!r}" ) From 6f727eac600ca44115d10e0bc864ea7713227967 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 12:04:39 -0400 Subject: [PATCH 3/8] Document the ESI premium stage (#454) docs/us-esi-employer-premiums.md: what the two inputs feed in policyengine-us 2.2.1, the source fields, the cell method, the BEA anchor and why NHE is only an upper bound, the pre-tax share assumptions, the measurements on both pinned pools, the gates, lineage, and what the change does not do. Adds the six restored columns to the Census person-column review, the operator to the base-build ordering note, and a changelog fragment. Co-Authored-By: Claude Opus 5.5 --- .../us-esi-employer-premiums-454.added.md | 1 + docs/us-asec-census-person-columns.md | 40 +++ docs/us-esi-employer-premiums.md | 255 ++++++++++++++++++ docs/us-multispine-operator-ordering.md | 2 +- 4 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 changelog.d/us-esi-employer-premiums-454.added.md create mode 100644 docs/us-esi-employer-premiums.md diff --git a/changelog.d/us-esi-employer-premiums-454.added.md b/changelog.d/us-esi-employer-premiums-454.added.md new file mode 100644 index 0000000000..e407062eea --- /dev/null +++ b/changelog.d/us-esi-employer-premiums-454.added.md @@ -0,0 +1 @@ +The US base build produces `employer_sponsored_insurance_premiums` and `pre_tax_health_insurance_premiums` (microcosm #454). The `meps_esi_premiums` stage, declared but unable to run because the pinned ASEC inputs lack the Census policyholder fields, now reads `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP2`, `PEMLR` and `NOEMP` restored from the pinned Census person files. Each employed current ESI policyholder takes the MEPS-IC 2025 employer share for their coverage tier, firm size and State (census division for government employers), scaled so the weighted total equals BEA NIPA Table 7.8 line 17 ($977.0B for 2024); dependents, retirees and the unincorporated self-employed carry zero. An eligible worker's reported premium is routed through payroll at the MEPS-IC firm-size pretax-offer rate. A release must now carry both inputs with signal and hold the employer total within 5% of the BEA anchor and at or below NHE Table 24, on the dense and the sparse export; the verdict ships as the `esi_premiums` gate evidence. The multispine pool does not run the stage yet. diff --git a/docs/us-asec-census-person-columns.md b/docs/us-asec-census-person-columns.md index 9eef04b40a..11cd9aeefc 100644 --- a/docs/us-asec-census-person-columns.md +++ b/docs/us-asec-census-person-columns.md @@ -198,3 +198,43 @@ the Census recode. Census recode restored, `A_EXPRRP == 13` now appears in every vintage, but it still conflates partners and roommates, so the exclusion's conclusion stands. + +## Columns added for the ESI premium stage (#454) + +Added 2026-10-10. The `meps_esi_premiums` stage +(`docs/us-esi-employer-premiums.md`) reads six Census person columns that +**none** of the four pinned inputs carries, the 2024 and 2025 files included, +so the restoration appends them for every pooled vintage and never has an +input copy to verify against. + +| column | restored | reader | +|---|---|---| +| `NOW_OWNGRP` | yes | `esi_premiums._esi_person_codes` → the current employment-based policyholder | +| `NOW_HIPAID` | yes | same → employer paid all, some or none of the premium | +| `NOW_GRPFTYP2` | yes | same → coverage tier (family, self plus one, self-only) | +| `NOW_GRPFTYP` | yes | same → cross-checked against `NOW_GRPFTYP2` | +| `PEMLR` | yes | same → employed at interview | +| `NOEMP` | yes | same → employer size band | + +The reviewed domains are the Census API codebooks for survey years 2023 to +2026 (`api.census.gov/data//cps/asec/mar/variables/.json`), +which are identical across the four years for each column. The data also +carry `PEMLR == 0` (children and Armed Forces), which the codebooks omit, so +the `PEMLR` domain is 0 to 7. + +Observed in the pinned person members (rows, by exact `PERIDNUM` join): + +| income year | member | persons | policyholders (`NOW_OWNGRP` 1) | `NOW_HIPAID` all / some / none | `NOW_GRPFTYP2` family / self plus one / self-only | employed (`PEMLR` 1-2) | +|---|---|---:|---:|---|---|---:| +| 2022 | `pppub23.csv` | 146,133 | 39,838 | 8,470 / 27,889 / 3,479 | 10,677 / 7,652 / 21,509 | 68,695 | +| 2023 | `pppub24.csv` | 144,265 | 39,161 | 7,843 / 28,047 / 3,271 | 10,393 / 7,386 / 21,382 | 68,511 | +| 2024 | `pppub25.csv` | 142,125 | 38,554 | 7,748 / 27,378 / 3,428 | 10,176 / 7,228 / 21,150 | 67,436 | +| 2025 | `pppub26.csv` | 134,729 | 36,185 | 7,449 / 25,717 / 3,019 | 9,623 / 6,806 / 19,756 | 63,429 | + +In every member the payment-status and tier counts sum to the policyholder +count, and everyone else carries 0 in both. The stage refuses a frame where +that does not hold. + +The older note that the 2024 and 2025 inputs "carry all 18" `NOW_*` recodes +counts the coverage-type recodes reviewed for #720. `NOW_OWNGRP`, +`NOW_HIPAID`, `NOW_GRPFTYP` and `NOW_GRPFTYP2` were never among them. diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md new file mode 100644 index 0000000000..0911bae200 --- /dev/null +++ b/docs/us-esi-employer-premiums.md @@ -0,0 +1,255 @@ +# US employer-sponsored insurance premiums (#454) + +Status as of 2026-10-10. Code: `microcosm.build.us_runtime.esi_premiums` +(stage `meps_esi_premiums`). Receipts: `experiments/us-esi-454/`. + +## What the stage produces + +Two PolicyEngine-US person inputs that no Microcosm stage produced. Their +consumers were read in the locked engine, policyengine-us 2.2.1: + +| Input | Engine consumer | +|---|---| +| `employer_sponsored_insurance_premiums` | `gov.household.cbo_market_income_additions`, summed into `cbo_household_market_income`. No other variable or parameter list names it. | +| `pre_tax_health_insurance_premiums` | `gov.irs.gross_income.pre_tax_contributions`, subtracted from `employment_income` in `irs_employment_income`; and `gov.irs.gross_income.fica_pre_tax_contributions`, subtracted in `payroll_tax_gross_wages`. | + +`tests/engine_contract/us/test_us_esi_premiums_engine.py` computes both in the +engine for one worker: a $2,000 pre-tax premium lowers `irs_employment_income`, +`payroll_tax_gross_wages` and adjusted gross income by exactly $2,000, and a +$10,000 employer premium raises `cbo_household_market_income` by exactly +$10,000 and moves none of the tax measures. + +The first was a recorded parity gap: the retired eCPS derivation (archived +`42ed5d45`, `datasets/cps/cps.py` L197-271) read the CPS ASEC current +employment-based coverage fields, and every pinned processed ASEC input omits +them. The second never had a producer. + +## Source fields + +`asec_census_person_columns` now restores six more Census person columns into +every pooled vintage, by exact one-to-one `PERIDNUM` identity from the pinned +official person members (`docs/us-asec-census-person-columns.md`). None of the +four pinned inputs (income years 2022 to 2025) carries any of them. + +| Column | Use | Codes | +|---|---|---| +| `NOW_OWNGRP` | current employment-based policyholder | 1 yes, 2 no, 0 out of universe | +| `NOW_HIPAID` | employer paid the premium | 1 all, 2 some, 3 none, 0 out of universe | +| `NOW_GRPFTYP2` | coverage tier | 1 family, 2 self plus one, 3 self-only | +| `NOW_GRPFTYP` | cross-check on the tier | 1 family, 2 self-only | +| `PEMLR` | employed at interview | 1 at work, 2 absent | +| `NOEMP` | employer size (the work-experience item "persons who work for employer") | 1 under 10, 2 10-49, 3 50-99, 4 100-499, 5 500-999, 6 1000+, 0 not in universe | + +The codes are the Census API codebooks for survey years 2023 to 2026, which +are identical across the four years for all six columns, and were checked +against the four pinned person members (`pppub23` to `pppub26`). In each file: + +- a policyholder always reports a payment status and a tier, and nobody else + reports either; +- `NOW_GRPFTYP` family equals `NOW_GRPFTYP2` family or self plus one, row for + row; +- a policyholder always has `NOW_GRP == 1`. + +The stage refuses any frame that breaks these relations, carries a missing or +out-of-codebook value, or lacks a column. It never proxies (`has_esi` and +`NOW_GRP` do not identify the policyholder) and never defaults. + +`PEIO1COW` (class of worker), `PHIP_VAL` (premiums paid out of pocket last +year) and `WSAL_VAL` are in the pinned inputs already. State is the household +`GESTFIPS`. + +## Employer premium + +**Universe.** An employed (`PEMLR` 1 or 2) current policyholder +(`NOW_OWNGRP` 1) whose current job has an employer (`PEIO1COW` 1 to 6). +Everyone else carries zero: + +- dependents, because the policyholder carries the premium; +- non-employed policyholders, because retiree and COBRA coverage is not + compensation of a current job; +- the unincorporated self-employed and workers without pay (`PEIO1COW` 7 and + 8), who have no employer to pay a share. + +**Cells.** MEPS-IC average total premium and average employee contribution per +enrolled employee, by coverage tier: + +- Private employers (`PEIO1COW` 4 to 6): MEPS-IC 2025 Series II, by State and + firm size. `NOEMP` 1 and 2 take the under-50 column, 3 to 6 the 50-or-more + column, and 0 (not in universe) the State's all-sizes column. MEPS-IC's + finer bands (10-24, 25-99, 100-999) do not nest inside the `NOEMP` bands, so + they cannot be used. +- State government (`PEIO1COW` 2): MEPS-IC 2024 Series III, State-government + column, by census division. +- Local and federal government (`PEIO1COW` 3 and 1): the same table's + all-governments column. MEPS-IC does not survey the federal government; this + is a stand-in. + +Series III for 2025 is not yet published. The 2024 government cells are aged +to 2025 by the private-sector national ratio of the same tier and measure +(for example single premium, 9,025 / 8,486). + +AHRQ suppresses nine small-firm employee-contribution cells in the 2025 State +tables (family: AK, DE, NM, SC, WV; self plus one: AK, KY, SC, SD). Each takes +the national small-firm cell times the State's all-sizes level relative to the +nation. No premium cell the stage reads is suppressed. + +**Share.** Employer paid all (`NOW_HIPAID` 1): the cell premium. Some (2): +premium minus employee contribution. None (3): zero. + +**Scale.** One factor scales every share so that the weighted total equals the +anchor for the build year. + +## The anchor + +| Series | CY2024 | Concept | +|---|---:|---| +| BEA NIPA Table 7.8 line 17, B4923C, "Private group health insurance" | **$977.0B** | Employer contributions as a supplement to wages and salaries, including government employers' contributions to privately administered plans. The same series is NIPA Table 6.11D line 32, and BEA's handbook (chapter 10, table 10.B, line 32) says its private and State and local estimates come from MEPS data "on insurance purchased by employers for employees". | +| CMS NHE Table 24, employer contribution to ESI premiums | $1,047.0B | The NHEA methodology paper (2024, p. 28) says the ESI estimates cover "active employees, continuation of health coverage (COBRA), and retirees"; note 1 of the table adds Medicare Retiree Drug Subsidy payments to the private-employer line. | + +The stage scales to **BEA**, the concept that matches an employed-policyholder +column, and uses **NHE** as an upper bound. + +Two things changed from what the issue recorded in July: + +1. BEA revised 2024. The issue cited $1,002.9B, from the release of + 2025-09-26. The annual update published 2026-09-30 puts 2024 at $977.034B + (and 2025 at $1,027.929B). The stage pins the September 2026 file by + SHA-256. +2. The issue leaned toward NHE as "the broadest match to a person-level + employer-paid concept". That held for a column that includes retirees. This + column excludes them, and the measurement below shows NHE's extra mass is + the non-employed policyholders. + +**Measured concept check.** Applying the same cells to *every* current +policyholder, employed or not, reproduces NHE: + +| Pool (income years) | All policyholders, raw | Ratio to NHE | Employed universe, raw | Non-employed or no-employer share | +|---|---:|---:|---:|---:| +| 2023-2025 (default) | $1,031.0B | 0.985 | $912.8B | 11.5% | +| 2022-2024 | $1,036.8B | 0.990 | $916.2B | 11.6% | + +So scaling an employed-only column to NHE would move about $70B of retiree and +COBRA premiums onto workers. Non-employed policyholders are priced at +active-employee cells in this check, which overstates Medicare-age retirees +whose employer plans are supplements; that is why it is a concept check and +not the column. + +## Pre-tax employee premium + +**Eligible.** A person in the employer-premium universe with wages last year +(`WSAL_VAL` > 0), a reported premium (`PHIP_VAL` > 0) and an employee share to +pay (`NOW_HIPAID` 2 or 3). + +**Amount.** The reported premium, `PHIP_VAL`, or zero. + +**Probability.** The share of insurance-offering private establishments that +offer pretax employee contributions, by firm size: MEPS-IC 2025 Table I.A.2.j +over Table I.A.2. + +| Firm size | Offer pretax contributions | Offer health insurance | Share used | +|---|---:|---:|---:| +| Under 50 | 15.9% | 32.1% | 49.5% | +| 50 or more | 87.8% | 94.8% | 92.6% | +| All (size unknown) | 35.5% | 49.2% | 72.2% | + +The draw is a seeded blake2b uniform keyed by source identity, so the two +support clones of a person agree and a rerun is bit-reproducible. + +Assumptions, each a known limit: + +- Table I.A.2.j's denominator is all establishments. Its title does not say + "that offer health insurance", unlike its neighbors I.A.2.a to I.A.2.i, so + the stage divides by the offer rate. This assumes pretax premium + contributions exist only where insurance is offered. +- The rates are establishment-weighted. Large establishments hold most + enrollees and offer pretax contributions more often, so the column is a + lower estimate. +- Government employers take the private firm-size rate for their `NOEMP`. +- `PHIP_VAL` covers every premium the person paid last year, not only the ESI + employee share. +- Coverage is measured at interview and the premium over the prior calendar + year. + +## Measurements on the pinned inputs + +`experiments/us-esi-454/run_stage_on_pinned_pool.py` pools the real pinned +inputs the way the base build does and runs the stage. These are +pre-calibration figures at pooled ASEC weights, target year 2024. + +| | 2023-2025 (default) | 2022-2024 | +|---|---:|---:| +| Current ESI policyholders | 92.4M | 92.9M | +| Employed, with an employer | 79.7M | 79.8M | +| With a positive employer premium | 74.0M | 74.1M | +| Raw MEPS-IC total | $912.8B | $916.2B | +| Scale factor | 1.070 | 1.066 | +| Employer premium total | $977.0B | $977.0B | +| Mean per policyholder with a positive premium | **$13,209** | $13,193 | +| Mean per employed policyholder | $12,260 | $12,245 | +| By tier: family / self plus one / self-only | $414.7B / $208.0B / $354.3B | $415.7B / $210.1B / $351.3B | +| By employer: private / State and local / federal | $756.7B / $180.7B / $39.6B | $756.0B / $180.8B / $40.2B | +| Pre-tax premiums | $181.1B over 52.5M | $174.6B over 52.5M | +| Share of eligible workers selected | 86.5% | 86.6% | + +The private-employer total ($756.7B) sits beside NHE's private line ($753.4B). +The State and local total is well under NHE's $247.0B, which includes +government retirees. + +Both the signal gate and the anchor gate pass on each pool, before and after +the PUF-support clone, and the clone conserves both totals. + +## Gates + +| Gate | Where | Checks | +|---|---|---| +| `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Columns present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible. With the raw ASEC columns still on the frame: no employer premium outside the universe or where the employer pays none, every premium is one common multiple of its MEPS-IC share, the raw share per holder is between $6k and $20k (a broken cell table cannot hide behind the scale factor), and every pre-tax premium is the reported premium of an eligible person. | +| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | Either column absent or zero-mass is red. The employer premium total must be within **5%** of BEA for the release period and at or below NHE's employer contribution. Pre-tax premiums must be at or below NHE's employee contribution ($382.1B). | +| `us_release_input_coverage_gate` | release tool | Both inputs are `required` with no reviewed exclusion, so an export that drops or flattens either fails. | +| reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B; neutralizing the pre-tax premium must raise `income_tax` by at least $5B. | + +The 5% tolerance is wider than BEA's own revision of 2024 between vintages +(2.6%) and narrower than the NHE-BEA gap (7.2%), so a release that drifts to +the retiree-inclusive concept fails. + +The stage pins the pre-calibration total exactly. Nothing calibrates the +column, so the anchor gate is what holds a release's total after reweighting +and sparse selection. The anchor gate's verdict ships as the `esi_premiums` +gate-evidence artifact (`esi_premiums.json`) and both manifests record it. +`--allow-esi-premium-gaps` records the gates without failing a diagnostic +build, for example on a base built before this stage. + +## Lineage + +- Census person members: pinned in + `education_assistance_source.ASEC_EDUCATION_ASSISTANCE_ARCHIVES` (archive + and member SHA-256), read by the #720 restoration. +- MEPS-IC cells: four AHRQ PDFs pinned by URL, byte length and SHA-256 in + `tools/build_us_meps_ic_esi_cells.py`, extracted to + `us_runtime/data/meps_ic_esi_premium_cells.json`, whose own SHA-256 the + stage pins and checks on load. +- BEA and NHE files: URL and SHA-256 in the stage manifest + (`us/source_stages.json`) and in the module. +- The base summary records the stage's full summary twice + (`esi_premiums_signal`, `esi_premiums_post_clone_signal`): scale factor, raw + and scaled totals, totals by tier and employer sector, weighted + policyholder counts, the anchor and the cell-table digest. + +## What this does not do + +- **The multispine pool.** `multispine_pool.py` runs its own operator chain on + CPS-source rows that carry only part of the stacked population's weight, so + scaling to a national total there needs its own design. Until then the + coverage and anchor gates are red on that lineage. +- **A calibration target.** The pinned feed banks only the NHE series. The + target-parity manifest keeps the two `cms_nhe` ESI families as `deferred` + reviewed exclusions and records BEA's series as a source-absent family. + Banking B4923C in the ledger would let the column be calibrated, not only + gated. +- **The engine's medical-expense base.** In policyengine-us 2.2.1, + `medical_expense_health_insurance_premiums` counts the whole reported + premium (`health_insurance_premiums_without_medicare_part_b`), so a premium + marked pre-tax here also sits in the itemized medical-expense base. +- **The wage targets.** `fiscal_targets.SOI_AMOUNT_MEASURE_VARIABLES` maps SOI + wages and salaries to `employment_income`, while the engine taxes + `irs_employment_income`, which is net of pre-tax contributions. This stage + widens that existing gap by the pre-tax premium total. diff --git a/docs/us-multispine-operator-ordering.md b/docs/us-multispine-operator-ordering.md index f3f62e66aa..507b2dd70c 100644 --- a/docs/us-multispine-operator-ordering.md +++ b/docs/us-multispine-operator-ordering.md @@ -23,7 +23,7 @@ sequence. | Boundary | Operators | State consumed | |---|---|---| | `source_construction` | `_load_base_frame_from_args` | Either an existing US base H5 or pooled ASEC unit frames built from the declared source years. | -| `pre_clone_enrichment` | `derive_us_cps_carried_inputs`; prior-year income, relationships, Medicare take-up, housing, eligibility, pregnancy, WIC, child support, disability benefits, workers compensation, weeks unemployed, childcare, energy subsidy, retirement contribution/distribution, and immigration operators | The ASEC-only frame, build year and seed; some operators also consume pinned external source tables. The adjacent-year income join and rent draw happen here, before row expansion. Housing can consume the pinned ACS 2022 rent donor when its input gate is not already satisfied. | +| `pre_clone_enrichment` | `derive_us_cps_carried_inputs`; prior-year income, relationships, Medicare take-up, housing, eligibility, pregnancy, WIC, child support, disability benefits, workers compensation, weeks unemployed, childcare, energy subsidy, retirement contribution/distribution, ESI premiums (#454), and immigration operators | The ASEC-only frame, build year and seed; some operators also consume pinned external source tables. The adjacent-year income join and rent draw happen here, before row expansion. Housing can consume the pinned ACS 2022 rent donor when its input gate is not already satisfied. | | `clone_feature_extraction` | `clone_us_frame_for_puf_support`; PUF donor extraction; primary-QRF initialization | The already enriched ASEC frame and processed PUF donor arrays. Cloning creates an ASEC copy and a PUF-tax-detail copy and splits weights across them. | | `primary_qrf_chain` and `qrf_finalization` | Chained weighted QRF fits and finalization | Predictors on the cloned frame, PUF donor targets, design weights, fit seed, and estimator count. Predictions are assigned to the PUF-detail copy. | | PUF tail and derived-detail stages | Capital-gains tail transfer, capital-gain distributions, QBI reconciliation | The cloned, QRF-imputed frame plus PUF donor detail and declared deterministic reconciliation rules. | From 6464019537b4a71eae2cef07d9d6914bda6fa473 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 19:58:02 -0400 Subject: [PATCH 4/8] Rework the ESI premium stage after independent review (#454) The first independent review asked for changes on five points; each held up against sources. Anchor and universe. BEA NIPA 7.8 line 17 is not an active-worker series: BEA's State Personal Income methodology (December 2025, paragraphs 3.18 and 3.19) says it covers active and retired employees, as CMS NHE Table 24 does. The stage now prices every current policyholder, scales that universe to NHE Table 24 (the anchor #454 chose), and writes the column for employed policyholders only. Retiree and COBRA dollars are priced and left out, not loaded onto workers. BEA and MEPS-IC's own private-sector enrollment total are recorded beside the anchor verdict. Payment status. "Employer pays some" subtracts the contribution of an employee who pays one: MEPS-IC's average contribution divided by one minus the published share of enrollees with no contribution (Tables II.C/D/E.4.a). A cohort with MEPS-IC's own mix reproduces the published cell mean. Gates. Both gates fail when a raw ASEC coverage column is missing. The anchor gate recomputes the anchor-universe total at release weights and bands the employed share. The evidence file carries the base and export signal verdicts, so a waived failure stays on the record. An ACS local build refuses a frame whose input is null on some rows instead of default-filling it. The output family is registered at the raw-stage operator boundary. Pre-tax premium. Removed from this change. PolicyEngine-US treats the premium inputs as disjoint (PolicyEngine/policyengine-us#10046), so a producer must move those dollars out of the reported premium on an engine that includes that change. The earlier derivation is kept on branch esi-pre-tax-premiums-454. The stage now has one output and no seeded draw; the seed-protocol registration is reverted. Measured on the default pinned pool, target year 2024, before calibration: $925.7B over 74.0M employed policyholders with a positive premium, mean $12,515. Receipts regenerated for both pools. Co-Authored-By: Claude Opus 5.5 --- .../us-esi-employer-premiums-454.added.md | 2 +- docs/evidence/spec-engine/us-f0-coverage.json | 93 +- docs/us-esi-employer-premiums.md | 366 +++++--- experiments/us-esi-454/README.md | 9 +- .../receipts/stage_on_pool_2022_2024.json | 209 +++-- .../receipts/stage_on_pool_2023_2025.json | 211 +++-- .../us-esi-454/run_stage_on_pinned_pool.py | 87 +- .../src/microcosm/build/source_manifest.py | 1 - .../build/spec_engine/field_usage.py | 22 +- .../build/spec_engine/inventory_coverage.py | 33 +- .../spec_engine/schema/sources.schema.json | 64 +- .../src/microcosm/build/spec_engine/seeds.py | 10 - .../us/release_input_coverage_manifest.json | 30 +- .../src/microcosm/build/us/source_stages.json | 69 +- .../src/microcosm/build/us/spec/sources.yaml | 129 ++- .../src/microcosm/build/us/spec/spine.yaml | 4 - .../build/us/target_parity_manifest.json | 20 +- .../microcosm/build/us_runtime/__init__.py | 14 +- .../data/MEPS_IC_ESI_PREMIUM_CELLS.md | 12 +- .../data/meps_ic_esi_premium_cells.json | 142 ++- .../build/us_runtime/esi_premiums.py | 754 +++++++++------- .../build/us_runtime/operator_boundary.py | 6 + .../us_runtime/release_input_coverage.py | 9 +- .../build/us_runtime/source_runtime.py | 4 - .../us/test_spec_engine_coverage_tool.py | 18 +- .../us/test_us_esi_premiums_engine.py | 56 +- .../shared/test_spec_engine_compiler_ir.py | 8 +- .../shared/test_spec_engine_field_usage.py | 16 +- .../test_spec_engine_identity_contracts.py | 4 +- .../shared/test_spec_engine_loader.py | 2 +- .../shared/test_spec_engine_plan_lock.py | 2 +- .../shared/test_spec_engine_seeds.py | 2 - .../us/test_us_acs_local_release_tool.py | 46 + .../us/test_us_bundle_core_contracts.py | 2 +- .../engine_free/us/test_us_esi_premiums.py | 837 ++++++++++++------ .../us/test_us_fiscal_refresh_builder.py | 29 +- .../engine_free/us/test_us_spec_bundle.py | 2 +- .../engine_free/us/test_us_spine_blindness.py | 4 +- .../us/test_us_multispine_pool_tool.py | 2 +- .../spec_engine_inventory_coverage.py | 6 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_acs_local_release.py | 11 +- tools/build_us_fiscal_refresh_release.py | 53 +- tools/build_us_meps_ic_esi_cells.py | 147 +-- ...uild_us_release_input_coverage_manifest.py | 70 +- tools/build_us_target_parity_manifest.py | 68 +- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 2 +- .../identity_contracts.py | 7 +- 49 files changed, 2177 insertions(+), 1521 deletions(-) diff --git a/changelog.d/us-esi-employer-premiums-454.added.md b/changelog.d/us-esi-employer-premiums-454.added.md index e407062eea..25434896b9 100644 --- a/changelog.d/us-esi-employer-premiums-454.added.md +++ b/changelog.d/us-esi-employer-premiums-454.added.md @@ -1 +1 @@ -The US base build produces `employer_sponsored_insurance_premiums` and `pre_tax_health_insurance_premiums` (microcosm #454). The `meps_esi_premiums` stage, declared but unable to run because the pinned ASEC inputs lack the Census policyholder fields, now reads `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP2`, `PEMLR` and `NOEMP` restored from the pinned Census person files. Each employed current ESI policyholder takes the MEPS-IC 2025 employer share for their coverage tier, firm size and State (census division for government employers), scaled so the weighted total equals BEA NIPA Table 7.8 line 17 ($977.0B for 2024); dependents, retirees and the unincorporated self-employed carry zero. An eligible worker's reported premium is routed through payroll at the MEPS-IC firm-size pretax-offer rate. A release must now carry both inputs with signal and hold the employer total within 5% of the BEA anchor and at or below NHE Table 24, on the dense and the sparse export; the verdict ships as the `esi_premiums` gate evidence. The multispine pool does not run the stage yet. +The US base build produces `employer_sponsored_insurance_premiums` (microcosm #454). The `meps_esi_premiums` stage, declared but unable to run because the pinned ASEC inputs lack the Census policyholder fields, now reads `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP2`, `PEMLR` and `NOEMP` restored from the pinned Census person files. Every current ESI policyholder takes the MEPS-IC 2025 employer share for their coverage tier, firm size and State (census division for government employers), and one factor scales that whole universe to CMS NHE Table 24's employer contribution ($1,047.0B for 2024), which counts active employees, COBRA enrollees and retirees. The column carries the employed policyholders' part ($925.7B on the default pool, before calibration); dependents, non-employed policyholders and the unincorporated self-employed carry zero. A release must now carry the input with signal and its raw ASEC coverage columns, and hold the anchor-universe total within 5% of the anchor on the dense and the sparse export; the verdict ships as the `esi_premiums` gate evidence. The multispine pool does not run the stage yet, and an ACS local build refuses a frame whose input is null on its ACS rows. `pre_tax_health_insurance_premiums` is not produced: it waits for the engine's disjoint premium contract (policyengine-us #10046). diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 62fa7c72eb..ac9a0a167c 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "d16558ea01d0eef4edd936f4f5166d25411d054d610cacbc3cd6037ee2737c00", + "documentation_sha256": "f002c79cc4a9296d34573d9c259eea0058f06e999630752ae3396395333889eb", "field_usage": { - "authored_normative_field_count": 32452, + "authored_normative_field_count": 32436, "claim_count": 49, "claims": [ { @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1761, - "pointer_sha256": "bdecb4db8c7c0a6b1dac1b39b4490007627f83bee305abcc1456304fdbfb5258", + "pointer_count": 1748, + "pointer_sha256": "dcc891c12eb675ee7f21c903995db98c3b8c6ccba569d71f0fd59b589abce04c", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -544,8 +544,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 280, - "pointer_sha256": "6cb87c6d4e3cca9907cc85d78a0ebd391281dc70391d30111d57a8bc84848bb6", + "pointer_count": 277, + "pointer_sha256": "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/authored/spec~1spine.yaml/seed_site_bindings", @@ -736,8 +736,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 843, - "pointer_sha256": "c51302fee9ab90cd34097f3d6cc8a765f2a12cb236e68a5772b87d0756f45242", + "pointer_count": 826, + "pointer_sha256": "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_protocol", @@ -750,8 +750,8 @@ "legacy_sinks": [], "mode": "compiler_semantic", "pointer_class": "all", - "pointer_count": 280, - "pointer_sha256": "1f88dfbe50cabf8e57c36f717b0cc952b5c423c1c2d62ac4219ec3c3872ead9f", + "pointer_count": 277, + "pointer_sha256": "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", "rationale": null, "relative_sink_prefix": null, "source_prefix": "/resolved/seed_site_bindings", @@ -772,21 +772,21 @@ "verifier": "vintages" } ], - "configuration_field_count": 42252, - "consumed_field_count": 42252, + "configuration_field_count": 42216, + "consumed_field_count": 42216, "generation0_effect_counts": { - "legacy_behavior": 38543, - "no_generation0_effect": 3709 + "legacy_behavior": 38530, + "no_generation0_effect": 3686 }, "mode_counts": { - "compiler_semantic": 27740, + "compiler_semantic": 27717, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14055 + "legacy_behavior": 14042 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "8afbe29a5ad75630b11e561e242d1882c94efc10648d287e80c6ab0a7f44f1bc", - "resolved_binding_field_count": 9800, + "pointer_inventory_sha256": "e8d32da2dbdaa015cb4eb363084c66cf509a93dde76cd473113750120dcaef80", + "resolved_binding_field_count": 9780, "unused_field_count": 0 }, "inventory_coverage": { @@ -815,9 +815,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 113, - "seed_owner_rows": 55, - "seed_sites": 54, + "seed_owner_bindings": 112, + "seed_owner_rows": 54, + "seed_sites": 53, "seed_streams": 14, "source_operators": 16, "source_stages": 38, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "b4f9cbbee81eff7f59304a114ed105910140f0799797c87f26ae9eb5090060f8" + "sha256": "3bfce32d85079804f869686bd6bf983dbbb7e262c05a982a9d40d85ae0f69238" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "b4f9cbbee81eff7f59304a114ed105910140f0799797c87f26ae9eb5090060f8" + "sha256": "3bfce32d85079804f869686bd6bf983dbbb7e262c05a982a9d40d85ae0f69238" }, "status": "covered" }, @@ -1533,7 +1533,7 @@ "expected": { "groups": 12, "partition": "disjoint and exhaustive", - "sites": 54 + "sites": 53 }, "failures": [], "observed": { @@ -1548,7 +1548,6 @@ "source_joint_count_calibration", "snap_take_up_assignment", "pregnancy_assignment", - "pre_tax_health_premium_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", "immigration_ead_workers_assignment", @@ -1617,7 +1616,7 @@ "sipp_tip_training_cap" ] }, - "sites": 54 + "sites": 53 }, "status": "covered" }, @@ -1634,12 +1633,12 @@ "compiler_ir.seed_stream_map.owners" ], "expected": { - "owner_rows": 55, + "owner_rows": 54, "relation": "site-order-preserving owner aggregation" }, "failures": [], "observed": { - "owner_rows": 55 + "owner_rows": 54 }, "status": "covered" }, @@ -1657,13 +1656,13 @@ "compiler_ir.node_slices" ], "expected": { - "map_sha256": "bdb924d0b21e55e387a533d179dc6c0f7d239bee79bd9de640729af17c9b2301", - "protocol_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d" + "map_sha256": "5641418e4a815da24744d59dfcf82fb11aa2f2ddc290ffcab9627383ea6fa682", + "protocol_sha256": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb" }, "failures": [], "observed": { - "map_sha256": "bdb924d0b21e55e387a533d179dc6c0f7d239bee79bd9de640729af17c9b2301", - "protocol_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d" + "map_sha256": "5641418e4a815da24744d59dfcf82fb11aa2f2ddc290ffcab9627383ea6fa682", + "protocol_sha256": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb" }, "status": "covered" }, @@ -1678,7 +1677,7 @@ "compiler_ir.seed_stream_map" ], "expected": { - "implementation_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d", + "implementation_sha256": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1699,7 +1698,7 @@ }, "failures": [], "observed": { - "implementation_sha256": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d", + "implementation_sha256": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb", "protocol": "legacy-v1", "streams": [ "build_model", @@ -1733,12 +1732,12 @@ ], "expected": { "relation": "all site fields preserved exactly", - "sites": 54 + "sites": 53 }, "failures": [], "observed": { - "sha256": "a64c6425f6926eb072d260dddb60cacf5bd145c786a9b9bf68823ec45c077945", - "sites": 54 + "sha256": "a0366cd518f42a240b589af3ef6580aac44ad7ed70969bcc669b365d315146ab", + "sites": 53 }, "status": "covered" }, @@ -1753,12 +1752,12 @@ "compiler_ir.seed_stream_map.sites.owners" ], "expected": { - "bindings": 113, - "coverage": "all 54 sites" + "bindings": 112, + "coverage": "all 53 sites" }, "failures": [], "observed": { - "bindings": 113 + "bindings": 112 }, "status": "covered" }, @@ -1776,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "b7f2650fd828f593bb53c8f6d37cbf1e0318111b8a81d4bc279954ed25e33fcd", + "sha256": "b90f0b264f9357eeda5169416babb50f3f360927a70aa0fd6ec78aa54af50427", "stages": 37 }, "failures": [], "observed": { - "sha256": "b7f2650fd828f593bb53c8f6d37cbf1e0318111b8a81d4bc279954ed25e33fcd", + "sha256": "b90f0b264f9357eeda5169416babb50f3f360927a70aa0fd6ec78aa54af50427", "stages": 38 }, "status": "covered" @@ -1888,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "69942294839a762738095894b87f6e72ea4faac03ebc020a4406c4f27bbb5c40" + "sha256": "e2a40490a6b705bab15bf2e7dff8eea4d7841ad70d3b7858874778d4e4829021" }, "failures": [], "observed": { @@ -1912,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "69942294839a762738095894b87f6e72ea4faac03ebc020a4406c4f27bbb5c40" + "sha256": "e2a40490a6b705bab15bf2e7dff8eea4d7841ad70d3b7858874778d4e4829021" }, "status": "covered" }, @@ -1976,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "df681e94538c72b5ac39c4c3847fa86cf51eae8c38f545720344b3c94b58897a" + "sha256": "fa02861d576da88cc771651aa309fb87bd8ac9427157b4175bced510f20d6d4d" }, "status": "covered" }, @@ -2600,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e9360b25997322886028b7a8a8964007201e2b36b7bbd341d0b483093e026950" + "spec_sha256": "a1e98deaa1f5a451c7d0e1f380a7bf50a82b0cbc94753b885b7cf9a11e7d0005" } }, "report_schema_version": 3, @@ -2610,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e9360b25997322886028b7a8a8964007201e2b36b7bbd341d0b483093e026950" + "spec_sha256": "a1e98deaa1f5a451c7d0e1f380a7bf50a82b0cbc94753b885b7cf9a11e7d0005" }, "status": "pass" } diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md index 0911bae200..d1f2e30cad 100644 --- a/docs/us-esi-employer-premiums.md +++ b/docs/us-esi-employer-premiums.md @@ -5,24 +5,25 @@ Status as of 2026-10-10. Code: `microcosm.build.us_runtime.esi_premiums` ## What the stage produces -Two PolicyEngine-US person inputs that no Microcosm stage produced. Their -consumers were read in the locked engine, policyengine-us 2.2.1: - -| Input | Engine consumer | -|---|---| -| `employer_sponsored_insurance_premiums` | `gov.household.cbo_market_income_additions`, summed into `cbo_household_market_income`. No other variable or parameter list names it. | -| `pre_tax_health_insurance_premiums` | `gov.irs.gross_income.pre_tax_contributions`, subtracted from `employment_income` in `irs_employment_income`; and `gov.irs.gross_income.fica_pre_tax_contributions`, subtracted in `payroll_tax_gross_wages`. | - -`tests/engine_contract/us/test_us_esi_premiums_engine.py` computes both in the -engine for one worker: a $2,000 pre-tax premium lowers `irs_employment_income`, -`payroll_tax_gross_wages` and adjusted gross income by exactly $2,000, and a -$10,000 employer premium raises `cbo_household_market_income` by exactly -$10,000 and moves none of the tax measures. - -The first was a recorded parity gap: the retired eCPS derivation (archived +One PolicyEngine-US person input that no Microcosm stage produced, +`employer_sponsored_insurance_premiums`. Its consumer was read in the locked +engine, policyengine-us 2.2.1: `gov.household.cbo_market_income_additions`, +summed into `cbo_household_market_income`. No other variable or parameter +list names it. + +`tests/engine_contract/us/test_us_esi_premiums_engine.py` computes it in the +engine for one worker: a $10,000 employer premium raises +`cbo_household_market_income` by exactly $10,000 and moves none of the tax +measures. + +It was a recorded parity gap: the retired eCPS derivation (archived `42ed5d45`, `datasets/cps/cps.py` L197-271) read the CPS ASEC current employment-based coverage fields, and every pinned processed ASEC input omits -them. The second never had a producer. +them. + +The stage does not produce `pre_tax_health_insurance_premiums`, the employee +share paid by pre-tax payroll deduction. See "The pre-tax employee premium" +below. ## Source fields @@ -47,31 +48,47 @@ against the four pinned person members (`pppub23` to `pppub26`). In each file: - a policyholder always reports a payment status and a tier, and nobody else reports either; - `NOW_GRPFTYP` family equals `NOW_GRPFTYP2` family or self plus one, row for - row; -- a policyholder always has `NOW_GRP == 1`. + row. -The stage refuses any frame that breaks these relations, carries a missing or -out-of-codebook value, or lacks a column. It never proxies (`has_esi` and -`NOW_GRP` do not identify the policyholder) and never defaults. +The stage refuses any frame that breaks these two relations, carries a missing +or out-of-codebook value, or lacks a column. It never defaults a value. It +does not read `NOW_GRP` or `has_esi`: neither identifies the policyholder. (In +the four files every policyholder also has `NOW_GRP == 1`; that was observed, +and the stage does not enforce it.) -`PEIO1COW` (class of worker), `PHIP_VAL` (premiums paid out of pocket last -year) and `WSAL_VAL` are in the pinned inputs already. State is the household -`GESTFIPS`. +`PEIO1COW` (class of worker) is in the pinned inputs already. State is the +household `GESTFIPS`. ## Employer premium -**Universe.** An employed (`PEMLR` 1 or 2) current policyholder -(`NOW_OWNGRP` 1) whose current job has an employer (`PEIO1COW` 1 to 6). -Everyone else carries zero: +### Two universes + +The national accounts that publish an employer-contribution total count +contributions for active **and** retired employees (see the anchor section). +The stage therefore keeps two universes apart: + +- **Anchor universe: every current policyholder** (`NOW_OWNGRP` 1), employed + or not. Each is priced from MEPS-IC, and one factor scales this universe to + the anchor. +- **Column universe: employed policyholders with an employer** (`PEMLR` 1 or + 2, `PEIO1COW` 1 to 6). Only they carry + `employer_sponsored_insurance_premiums`, at the same factor. + +Everyone else carries zero in the column: - dependents, because the policyholder carries the premium; -- non-employed policyholders, because retiree and COBRA coverage is not - compensation of a current job; +- retired, unemployed and other non-employed policyholders, because their + coverage is not compensation of a current job; - the unincorporated self-employed and workers without pay (`PEIO1COW` 7 and 8), who have no employer to pay a share. -**Cells.** MEPS-IC average total premium and average employee contribution per -enrolled employee, by coverage tier: +The non-employed policyholders' scaled share is priced and then left out. It is +not loaded onto workers. + +### Cells + +MEPS-IC average total premium and average employee contribution per enrolled +employee, by coverage tier: - Private employers (`PEIO1COW` 4 to 6): MEPS-IC 2025 Series II, by State and firm size. `NOEMP` 1 and 2 take the under-50 column, 3 to 6 the 50-or-more @@ -83,6 +100,9 @@ enrolled employee, by coverage tier: - Local and federal government (`PEIO1COW` 3 and 1): the same table's all-governments column. MEPS-IC does not survey the federal government; this is a stand-in. +- No employer class (`PEIO1COW` 0, 7 or 8, which is most non-employed + policyholders): the State's all-sizes private column. A non-employed + policyholder who does report a class of worker keeps that employer's cell. Series III for 2025 is not yet published. The 2024 government cells are aged to 2025 by the private-sector national ratio of the same tier and measure @@ -93,82 +113,137 @@ tables (family: AK, DE, NM, SC, WV; self plus one: AK, KY, SC, SD). Each takes the national small-firm cell times the State's all-sizes level relative to the nation. No premium cell the stage reads is suppressed. -**Share.** Employer paid all (`NOW_HIPAID` 1): the cell premium. Some (2): -premium minus employee contribution. None (3): zero. - -**Scale.** One factor scales every share so that the weighted total equals the -anchor for the build year. +No MEPS-IC table prices retiree coverage, so a non-employed policyholder is +priced as an active employee. If retiree plans cost less than active plans, +the employed column is understated; the receipts carry the sensitivity. -## The anchor +### Share by payment status -| Series | CY2024 | Concept | -|---|---:|---| -| BEA NIPA Table 7.8 line 17, B4923C, "Private group health insurance" | **$977.0B** | Employer contributions as a supplement to wages and salaries, including government employers' contributions to privately administered plans. The same series is NIPA Table 6.11D line 32, and BEA's handbook (chapter 10, table 10.B, line 32) says its private and State and local estimates come from MEPS data "on insurance purchased by employers for employees". | -| CMS NHE Table 24, employer contribution to ESI premiums | $1,047.0B | The NHEA methodology paper (2024, p. 28) says the ESI estimates cover "active employees, continuation of health coverage (COBRA), and retirees"; note 1 of the table adds Medicare Retiree Drug Subsidy payments to the private-employer line. | +- Employer paid all (`NOW_HIPAID` 1): the cell premium. +- Employer paid some (2): the premium minus the contribution of an employee + who pays one. +- Employer paid none (3): zero. -The stage scales to **BEA**, the concept that matches an employed-policyholder -column, and uses **NHE** as an upper bound. +MEPS-IC's average employee contribution is taken over every enrollee, +including those whose coverage required no contribution. Subtracting it from +"paid some" holders only, while "paid all" holders keep the whole premium, +would overstate the cell's employer mean. The stage divides the average +contribution by one minus the published share of enrollees with no +contribution (MEPS-IC 2025 Tables II.C.4.a, II.D.4.a and II.E.4.a, United +States row by firm size): -Two things changed from what the issue recorded in July: +| Tier | All sizes | Under 50 | 50 or more | +|---|---:|---:|---:| +| Self-only | 13.0% | 35.5% | 8.7% | +| Family | 7.7% | 28.6% | 4.6% | +| Self plus one | 6.0% | 23.1% | 3.6% | + +A cohort with MEPS-IC's own mix of the two groups then reproduces the cell's +published employer mean (premium minus average contribution) exactly. For the +national under-50 self-only cell, premium $9,034 and average contribution +$1,924, the unconditional rule gives $7,793 and the conditional rule the +published $7,110. + +Limits of this rule: + +- The no-contribution shares are national. Most State cells of those tables + are flagged unreliable or suppressed. +- Series III publishes no such share, so government cells take the private + 50-or-more share. +- CPS respondents say "employer paid all" more often than MEPS-IC + establishments report a zero contribution (18.7% of employed policyholders, + against the shares above). On the pinned pools the stage's raw mean among + employer-paying workers is therefore 3.1% above the published cell means + (`raw_over_published_employer_mean`), before the national factor. +- MEPS-IC publishes no share of enrollees who pay the whole premium. The 7.2% + of employed policyholders who say their employer pays nothing carry zero and + are treated as outside the MEPS-IC averages. + +### Scale + +One factor scales every policyholder's share so that the weighted +anchor-universe total equals the anchor for the build year. The column is that +factor times the employed policyholders' shares. -1. BEA revised 2024. The issue cited $1,002.9B, from the release of - 2025-09-26. The annual update published 2026-09-30 puts 2024 at $977.034B - (and 2025 at $1,027.929B). The stage pins the September 2026 file by - SHA-256. -2. The issue leaned toward NHE as "the broadest match to a person-level - employer-paid concept". That held for a column that includes retirees. This - column excludes them, and the measurement below shows NHE's extra mass is - the non-employed policyholders. +## The anchor -**Measured concept check.** Applying the same cells to *every* current -policyholder, employed or not, reproduces NHE: +| Series | CY2024 | Who it covers | +|---|---:|---| +| CMS NHE Table 24, "Employer Contribution to Private Health Insurance Premiums" | **$1,047.0B** | Private, federal and State and local employers. The NHEA methodology paper (2024, p. 29) says the estimates cover "active employees, continuation of health coverage (COBRA), and retirees" of private and State and local sponsors, from MEPS-IC, and federal "employers, employees and retirees", from OPM. | +| BEA NIPA Table 7.8 line 17, B4923C, "Private group health insurance" | $977.0B | Employer contributions as a supplement to wages and salaries. BEA's State Personal Income methodology (December 2025, paragraphs 3.18 and 3.19) says its MEPS source "covers both health insurance purchased by employers for their active and retired employees" and that the federal estimate covers "active and retired federal civilian employees". | -| Pool (income years) | All policyholders, raw | Ratio to NHE | Employed universe, raw | Non-employed or no-employer share | -|---|---:|---:|---:|---:| -| 2023-2025 (default) | $1,031.0B | 0.985 | $912.8B | 11.5% | -| 2022-2024 | $1,036.8B | 0.990 | $916.2B | 11.6% | +Both series measure the same concept, and both include retirees. The stage +scales to **NHE**, the series the issue chose and the pinned feed banks +(`cms_nhe.cy2024.esi_employer_contribution_premiums`). **BEA** is recorded +beside every anchor verdict and is not gated: it is 6.7% below NHE for 2024. -So scaling an employed-only column to NHE would move about $70B of retiree and -COBRA premiums onto workers. Non-employed policyholders are priced at -active-employee cells in this check, which overstates Medicare-age retirees -whose employer plans are supplements; that is why it is a concept check and -not the column. +Two facts differ from what the issue recorded in July: -## Pre-tax employee premium +1. BEA revised 2024. The issue cited $1,002.9B, from the release of + 2025-09-26. The annual update published 2026-09-30 puts 2024 at $977.034B. + The stage pins the September 2026 file by SHA-256. +2. The issue described NHE as the broader series because it includes + retirees. BEA's does too. -**Eligible.** A person in the employer-premium universe with wages last year -(`WSAL_VAL` > 0), a reported premium (`PHIP_VAL` > 0) and an employee share to -pay (`NOW_HIPAID` 2 or 3). +NHE Table 24 ends at CY2024, so the stage refuses a 2025 build year. BEA's +2025 value does not stand in for it. -**Amount.** The reported premium, `PHIP_VAL`, or zero. +### What the choice moves -**Probability.** The share of insurance-offering private establishments that -offer pretax employee contributions, by firm size: MEPS-IC 2025 Table I.A.2.j -over Table I.A.2. +The receipts restate the default pool's column under each defensible reading. +The anchor and the treatment of non-employed policyholders are methodology +choices; the first row is what the stage builds. -| Firm size | Offer pretax contributions | Offer health insurance | Share used | -|---|---:|---:|---:| -| Under 50 | 15.9% | 32.1% | 49.5% | -| 50 or more | 87.8% | 94.8% | 92.6% | -| All (size unknown) | 35.5% | 49.2% | 72.2% | - -The draw is a seeded blake2b uniform keyed by source identity, so the two -support clones of a person agree and a rerun is bit-reproducible. - -Assumptions, each a known limit: - -- Table I.A.2.j's denominator is all establishments. Its title does not say - "that offer health insurance", unlike its neighbors I.A.2.a to I.A.2.i, so - the stage divides by the offer rate. This assumes pretax premium - contributions exist only where insurance is offered. -- The rates are establishment-weighted. Large establishments hold most - enrollees and offer pretax contributions more often, so the column is a - lower estimate. -- Government employers take the private firm-size rate for their `NOEMP`. -- `PHIP_VAL` covers every premium the person paid last year, not only the ESI - employee share. -- Coverage is measured at interview and the premium over the prior calendar - year. +| Reading | Employed column | Mean per policyholder with a positive premium | +|---|---:|---:| +| **NHE over every policyholder (as built)** | **$925.7B** | **$12,515** | +| NHE over every policyholder, 65-and-over non-employed at half price | $948.1B | $12,817 | +| BEA over every policyholder | $863.9B | $11,679 | +| NHE loaded onto employed policyholders only | $1,047.0B | $14,155 | +| BEA loaded onto employed policyholders only | $977.0B | $13,209 | + +One outside check bears on it. MEPS-IC's own national rows give the +private-sector employer total for enrolled, active employees: employees times +the share in establishments that offer insurance times the share enrolled +there, times each tier's share of enrollees and its premium less average +contribution. That is $668.6B for 2024 (65.6M enrollees) and $714.4B for 2025. +The private-sector part of the as-built column is $713.2B, 6.7% above the 2024 +figure; under the BEA reading it would be $665.6B, 0.4% below. The check +covers the private sector only (77% of the column) and shares the cells' +source, so it informs the choice without settling it. + +## The pre-tax employee premium + +`pre_tax_health_insurance_premiums` has no producer, and this stage does not +add one. + +PolicyEngine-US treats the premium inputs as disjoint: a premium paid by +pre-tax payroll deduction belongs only in `pre_tax_health_insurance_premiums`, +and `health_insurance_premiums_without_medicare_part_b`, +`other_health_insurance_premiums` and `health_insurance_premiums` hold the +premiums not paid that way (PolicyEngine/policyengine-us#10046). The base +carries the whole reported premium (`PHIP_VAL`) in the second set. A producer +therefore has to move the pre-tax dollars out of the reported premium, not +copy them: + +- Copying them would count the same premium twice. The engine would exclude + it from wages and still deduct it for itemizers with medical expenses above + the floor (for a filer with $60,000 of wages, $20,000 of other medical + expenses and a $2,000 premium, income tax of $4,445.74 against $4,685.74). +- Moving them is only right on an engine that counts the pre-tax input where + the reported premium was counted. policyengine-us 2.2.1 does not: its + `spm_unit_health_insurance_premiums` adds `other_health_insurance_premiums` + and the modeled premiums, not the pre-tax input, so the moved dollars would + drop out of SPM medical out-of-pocket expenses. #10046 adds it. +- Moving about $180 billion out of two premium inputs also changes what the + release tool's input-mass parity gate and the eCPS parity reference expect + of those columns. + +An earlier round of this work assigned the reported premium, capped at wages, +to eligible workers at MEPS-IC pretax-offer rates and measured $180.1 billion +over 52.5 million workers on the default pool. That code is on branch +`esi-pre-tax-premiums-454` and is the starting point for the follow-up, which +is sequenced after an engine pin that includes #10046. ## Measurements on the pinned inputs @@ -181,19 +256,22 @@ pre-calibration figures at pooled ASEC weights, target year 2024. | Current ESI policyholders | 92.4M | 92.9M | | Employed, with an employer | 79.7M | 79.8M | | With a positive employer premium | 74.0M | 74.1M | -| Raw MEPS-IC total | $912.8B | $916.2B | -| Scale factor | 1.070 | 1.066 | -| Employer premium total | $977.0B | $977.0B | -| Mean per policyholder with a positive premium | **$13,209** | $13,193 | -| Mean per employed policyholder | $12,260 | $12,245 | -| By tier: family / self plus one / self-only | $414.7B / $208.0B / $354.3B | $415.7B / $210.1B / $351.3B | -| By employer: private / State and local / federal | $756.7B / $180.7B / $39.6B | $756.0B / $180.8B / $40.2B | -| Pre-tax premiums | $181.1B over 52.5M | $174.6B over 52.5M | -| Share of eligible workers selected | 86.5% | 86.6% | - -The private-employer total ($756.7B) sits beside NHE's private line ($753.4B). -The State and local total is well under NHE's $247.0B, which includes -government retirees. +| Other policyholders (priced, outside the column) | 12.7M | 13.2M | +| Raw MEPS-IC total, every policyholder | $1,006.4B | $1,012.1B | +| Scale factor | 1.040 | 1.034 | +| Anchor-universe total | $1,047.0B | $1,047.0B | +| **Employer premium column** | **$925.7B** | $923.9B | +| Employed share of the anchor universe | 88.4% | 88.2% | +| Other policyholders' share, not in the column | $121.3B | $123.1B | +| Mean per policyholder with a positive premium | **$12,515** | $12,476 | +| Mean per employed policyholder | $11,616 | $11,579 | +| By tier: family / self plus one / self-only | $393.1B / $198.2B / $334.4B | $393.2B / $199.7B / $331.0B | +| By employer: private / State and local / federal | $713.2B / $174.3B / $38.1B | $711.2B / $174.1B / $38.6B | + +Of the 12.7M other policyholders in the default pool, 7.1M are retired +(`PEMLR` 5), 2.1M are otherwise out of the labor force (7), 0.9M are disabled +(6), 1.0M are unemployed (3 and 4), 1.5M are employed without an employer +class, and 0.1M have no labor-force status (0). Both the signal gate and the anchor gate pass on each pool, before and after the PUF-support clone, and the clone conserves both totals. @@ -202,54 +280,66 @@ the PUF-support clone, and the clone conserves both totals. | Gate | Where | Checks | |---|---|---| -| `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Columns present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible. With the raw ASEC columns still on the frame: no employer premium outside the universe or where the employer pays none, every premium is one common multiple of its MEPS-IC share, the raw share per holder is between $6k and $20k (a broken cell table cannot hide behind the scale factor), and every pre-tax premium is the reported premium of an eligible person. | -| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | Either column absent or zero-mass is red. The employer premium total must be within **5%** of BEA for the release period and at or below NHE's employer contribution. Pre-tax premiums must be at or below NHE's employee contribution ($382.1B). | -| `us_release_input_coverage_gate` | release tool | Both inputs are `required` with no reviewed exclusion, so an export that drops or flattens either fails. | -| reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B; neutralizing the pre-tax premium must raise `income_tax` by at least $5B. | - -The 5% tolerance is wider than BEA's own revision of 2024 between vintages -(2.6%) and narrower than the NHE-BEA gap (7.2%), so a release that drifts to -the retiree-inclusive concept fails. +| `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Column present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible; no employer premium outside the column universe or where the employer pays none; every premium is one common multiple of its MEPS-IC share. | +| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of NHE Table 24 for the release period. Employed policyholders must carry 80% to 95% of it. | +| `us_release_input_coverage_gate` | release tool | The input is `required` with no reviewed exclusion, so an export that drops or flattens it fails. | +| reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B. | + +What the gates need, and what they do not catch: + +- **They need the raw ASEC columns.** Both gates recompute the assignment + from `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP`, `NOW_GRPFTYP2`, `PEMLR`, + `NOEMP`, `PEIO1COW` and State. A frame missing any of them fails both: it + cannot be certified. Published release files carry raw ASEC person fields + today (`PEIO1COW`, `PHIP_VAL`, `WSAL_VAL`, `NOW_MCAID`); a release path + that dropped the restored columns would turn both gates red. +- **The raw-mean band catches unit errors only.** The signal gate requires + the unscaled share per holder to sit between $6k and $20k, which catches a + cell table in the wrong units. A cell table that is 20% low passes: the + scale factor absorbs it. A re-pinned table is checked against the PDFs by + `tools/build_us_meps_ic_esi_cells.py --check`, not by this gate. +- **The 5% tolerance is a chosen criterion.** No release has calibrated this + column yet, so the tolerance is not derived from measured drift. The first + release build records the drift in `esi_premiums.json`. The stage pins the pre-calibration total exactly. Nothing calibrates the column, so the anchor gate is what holds a release's total after reweighting -and sparse selection. The anchor gate's verdict ships as the `esi_premiums` -gate-evidence artifact (`esi_premiums.json`) and both manifests record it. -`--allow-esi-premium-gaps` records the gates without failing a diagnostic -build, for example on a base built before this stage. +and sparse selection. `esi_premiums.json` is required release evidence: it +carries the anchor verdict, both cross-checks and the base and export signal +verdicts, and both manifests record it. `--allow-esi-premium-gaps` stops the +gates failing a diagnostic build, for example on a base built before this +stage; the failures it waives are still written to that file. ## Lineage - Census person members: pinned in `education_assistance_source.ASEC_EDUCATION_ASSISTANCE_ARCHIVES` (archive and member SHA-256), read by the #720 restoration. -- MEPS-IC cells: four AHRQ PDFs pinned by URL, byte length and SHA-256 in +- MEPS-IC cells: three AHRQ PDFs pinned by URL, byte length and SHA-256 in `tools/build_us_meps_ic_esi_cells.py`, extracted to `us_runtime/data/meps_ic_esi_premium_cells.json`, whose own SHA-256 the stage pins and checks on load. -- BEA and NHE files: URL and SHA-256 in the stage manifest - (`us/source_stages.json`) and in the module. +- NHE and BEA files: URL, byte length and SHA-256 in the stage manifest + (`us/source_stages.json`), and URL and SHA-256 in the module. - The base summary records the stage's full summary twice (`esi_premiums_signal`, `esi_premiums_post_clone_signal`): scale factor, raw - and scaled totals, totals by tier and employer sector, weighted - policyholder counts, the anchor and the cell-table digest. + and scaled totals for both universes, totals by tier and employer sector, + weighted policyholder counts, the anchor, the cross-check and the cell-table + digest. ## What this does not do - **The multispine pool.** `multispine_pool.py` runs its own operator chain on CPS-source rows that carry only part of the stacked population's weight, so scaling to a national total there needs its own design. Until then the - coverage and anchor gates are red on that lineage. -- **A calibration target.** The pinned feed banks only the NHE series. The - target-parity manifest keeps the two `cms_nhe` ESI families as `deferred` - reviewed exclusions and records BEA's series as a source-absent family. - Banking B4923C in the ledger would let the column be calibrated, not only - gated. -- **The engine's medical-expense base.** In policyengine-us 2.2.1, - `medical_expense_health_insurance_premiums` counts the whole reported - premium (`health_insurance_premiums_without_medicare_part_b`), so a premium - marked pre-tax here also sits in the itemized medical-expense base. -- **The wage targets.** `fiscal_targets.SOI_AMOUNT_MEASURE_VARIABLES` maps SOI - wages and salaries to `employment_income`, while the engine taxes - `irs_employment_income`, which is net of pre-tax contributions. This stage - widens that existing gap by the pre-tax premium total. + coverage and anchor gates are red on that lineage. The raw-stage operator + boundary already refuses a source frame that carries the output. +- **ACS local releases.** An ACS spine pooled beside a donor that carries the + input has it null on its ACS rows. `fill_reviewed_nulls` refuses that frame + instead of default-filling it to zero, so this lineage does not build until + the input is assigned or transferred for every spine. +- **A calibration target.** The column is the employed part of the NHE fact, + so the fact cannot be compiled as a sum target on it. The target-parity + manifest keeps the two `cms_nhe` ESI families as `deferred` reviewed + exclusions and records BEA's series as a source-absent family. +- **The pre-tax employee premium.** See the section above. diff --git a/experiments/us-esi-454/README.md b/experiments/us-esi-454/README.md index 8f88435f22..84072aceb0 100644 --- a/experiments/us-esi-454/README.md +++ b/experiments/us-esi-454/README.md @@ -25,10 +25,11 @@ About 80 seconds per pool. Each receipt records the input digests, the columns the restoration added per vintage with their observed code counts, the stage summary (scale factor, raw -and scaled totals, totals by tier and employer sector, weighted policyholder -counts), totals by vintage, both gate verdicts before and after the clone, and -`nhe_concept_check`: the same MEPS-IC cells applied to every current -policyholder, which is how NHE Table 24's broader concept was measured. +and scaled totals for both universes, totals by tier and employer sector, +weighted policyholder counts), totals by vintage, both gate verdicts before +and after the clone with the BEA and MEPS-IC cross-checks, and +`other_policyholder_sensitivity`: the employed column under each reading of +the anchor and of how policyholders outside the column are priced. These are pre-calibration figures at pooled ASEC weights. They are evidence about the stage, not about a release: diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json index 0b013b3e1f..a7b9d0ac53 100644 --- a/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json +++ b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json @@ -2,42 +2,60 @@ "anchor_gate": { "details": { "anchor": { + "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "anchor_universe_employer_total": 1047000000000.0, + "anchor_value": 1047000000000.0, + "cross_check": { "label": "Private group health insurance", "line": 17, "published": "2026-09-30", + "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", "values": { - "2023": 923195000000.0, - "2024": 977034000000.0, - "2025": 1027929000000.0 + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 } }, - "anchor_value": 977034000000.0, - "cross_check": { - "employee_contribution": { - "2023": 357800000000.0, - "2024": 382100000000.0 - }, - "employer_contribution": { - "2023": 975700000000.0, - "2024": 1047000000000.0 - }, - "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "Table 24 Employer-Sponsored Private Health Insurance" + "cross_check_ratio": 1.0716106092520834, + "employed_share_band": [ + 0.8, + 0.95 + ], + "employed_share_of_anchor_universe": 0.8824300567320681, + "employer_premium_by_sector": { + "federal": 38621718711.29335, + "private": 711190949595.6248, + "state_and_local": 174091601091.55725 }, - "cross_check_employer_ratio": 0.9331747851002865, - "cross_check_pre_tax_ratio": 0.45697198844356723, + "employer_premium_mean_per_positive_person": 12475.548691763059, "employer_premium_positive_persons": 74057205.19599111, - "employer_premium_total": 977034000000.0, - "pre_tax_premium_total": 174608996784.28705, + "employer_premium_total": 923904269398.4752, + "other_policyholder_employer_total": 123095730601.5247, + "private_active_cross_check": { + "employer_premium_private_sector": 711190949595.6248, + "meps_ic_private_active_employer_total": 668573745863.8779, + "ratio": 1.0637434598582394 + }, "relative_error": 0.0, "relative_tolerance": 0.05, - "time_period": 2024 + "scale_factor": 1.034460355190118, + "time_period": 2024, + "weighted_employed_policyholders": 79790597.17620382, + "weighted_other_policyholders": 13156453.970184255, + "weighted_policyholders": 92947051.14638808 }, "failures": [], "passed": true @@ -45,23 +63,20 @@ "by_vintage": { "2022": { "employer_premium_positive_persons": 24684189.26891587, - "employer_premium_total": 328107608719.83057, - "mean_per_positive_person": 13292.217343877182, - "pre_tax_premium_total": 55854772150.73417, + "employer_premium_total": 310331531290.6173, + "mean_per_positive_person": 12572.077126365637, "weighted_persons": 108650789.68666667 }, "2023": { "employer_premium_positive_persons": 24692526.63040858, - "employer_premium_total": 325317655884.286, - "mean_per_positive_person": 13174.741522143822, - "pre_tax_premium_total": 58750093810.022896, + "employer_premium_total": 307546511529.20874, + "mean_per_positive_person": 12455.044237978813, "weighted_persons": 108650789.68666664 }, "2024": { "employer_premium_positive_persons": 24680489.296666663, - "employer_premium_total": 323608735395.8834, - "mean_per_positive_person": 13111.92543656701, - "pre_tax_premium_total": 60004130823.53, + "employer_premium_total": 306026226578.64923, + "mean_per_positive_person": 12399.520240467073, "weighted_persons": 108650789.68666667 } }, @@ -92,13 +107,39 @@ } ], "issue": "PolicyEngine/microcosm#454", - "nhe_concept_check": { - "nhe_table_24_employer_contribution_2024": 1047000000000.0, - "non_employed_share_of_all": 0.1162675199483739, - "raw_all_policyholders_over_nhe": 0.9902231741950678, - "raw_all_policyholders_total": 1036763663382.236, - "raw_employed_universe_total": 916221723468.1926, - "raw_non_employed_or_no_employer_total": 120541939914.04333 + "other_policyholder_sensitivity": { + "as_built_other_priced_as_active": { + "employer_premium_total": 923904269398.4752, + "mean_per_positive_person": 12475.548691763057 + }, + "bea_anchor_all_on_employed": { + "employer_premium_total": 977034000000.0001, + "mean_per_positive_person": 13192.963431637698 + }, + "bea_anchor_other_priced_as_active": { + "employer_premium_total": 862164168049.1594, + "mean_per_positive_person": 11641.867469444152 + }, + "other_65_plus_priced_at_half": { + "employer_premium_total": 946325497990.1058, + "mean_per_positive_person": 12778.304224223311 + }, + "other_not_priced_all_anchor_on_employed": { + "employer_premium_total": 1047000000000.0, + "mean_per_positive_person": 14137.71958081773 + }, + "raw_other_65_plus_share_of_other": 0.4030440846499319, + "weighted_other_policyholders_65_plus": 5861452.143127043, + "weighted_other_policyholders_by_pemlr": { + "0": 106424.49450739344, + "1": 1425715.020176738, + "2": 81652.39084999224, + "3": 221253.36609046787, + "4": 770435.206337095, + "5": 7347264.321874038, + "6": 986297.5838004182, + "7": 2217411.586548111 + } }, "person_rows": 432523, "pool_census_person_columns": [ @@ -492,10 +533,10 @@ ], "puf_support_clone": { "anchor_gate": { - "employer_premium_total": 977034000000.0, + "anchor_universe_employer_total": 1047000000000.0002, + "employer_premium_total": 923904269398.4753, "failures": [], - "passed": true, - "pre_tax_premium_total": 174608996784.28705 + "passed": true }, "person_rows": 865046, "signal_gate": { @@ -503,7 +544,7 @@ "passed": true } }, - "seconds": 47.4, + "seconds": 35.4, "seed": 0, "signal_gate": { "failures": [], @@ -511,80 +552,78 @@ }, "summary": { "anchor": { + "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "anchor_universe_employer_total": 1047000000000.0, + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "clone_disagreement_source_persons": 0, + "cross_check": { "label": "Private group health insurance", "line": 17, "published": "2026-09-30", + "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", "values": { - "2023": 923195000000.0, - "2024": 977034000000.0, - "2025": 1027929000000.0 + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 } }, - "cells_sha256": "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8", - "clone_disagreement_source_persons": 0, - "cross_check": { - "employee_contribution": { - "2023": 357800000000.0, - "2024": 382100000000.0 - }, - "employer_contribution": { - "2023": 975700000000.0, - "2024": 1047000000000.0 - }, - "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "Table 24 Employer-Sponsored Private Health Insurance" - }, + "employed_share_of_anchor_universe": 0.8824300567320681, "employer_premium_by_sector": { - "federal": 40180113357.65264, - "private": 756015257851.7373, - "state_and_local": 180838628790.61008 + "federal": 38621718711.29335, + "private": 711190949595.6248, + "state_and_local": 174091601091.55725 }, "employer_premium_by_tier": { - "employee_plus_one": 210094674350.32474, - "family": 415669227862.2854, - "single": 351270097787.38983 + "employee_plus_one": 199685222296.09024, + "family": 393209175914.58295, + "single": 331009871187.80206 }, - "employer_premium_mean_per_positive_person": 13192.963431637696, + "employer_premium_mean_per_positive_person": 12475.548691763059, "employer_premium_outside_universe_rows": 0, "employer_premium_positive_persons": 74057205.19599111, "employer_premium_positive_share": 0.2272025370134953, - "employer_premium_total": 977034000000.0, + "employer_premium_total": 923904269398.4752, "employer_premium_where_employer_pays_none_rows": 0, "employer_premium_without_raw_share_rows": 0, + "meps_ic_private_active_employer_total": { + "2024": 668573745863.8779, + "2025": 714391090330.8774 + }, "negative_rows": 0, "nonfinite_rows": 0, - "pre_tax_eligible_weight": 60688921.39918899, - "pre_tax_ineligible_rows": 0, - "pre_tax_not_reported_premium_rows": 0, - "pre_tax_premium_positive_persons": 52528029.87619302, - "pre_tax_premium_positive_share": 0.16115247153342174, - "pre_tax_premium_total": 174608996784.28705, - "pre_tax_selected_share_of_eligible": 0.8655291388469951, - "pre_tax_shares": { - "50plus": 0.9261603375527426, - "lt50": 0.4953271028037383, - "total": 0.7215447154471545 - }, + "other_policyholder_employer_total": 123095730601.5247, + "raw_anchor_universe_total": 1012121919169.708, "raw_employer_share_band": [ 6000.0, 20000.0 ], - "raw_employer_share_mean_per_positive_person": 12371.810697465933, - "raw_employer_share_total": 916221723468.1926, - "scale_factor": 1.0663728822119753, - "scale_factor_spread": 2.220446049250313e-16, + "raw_employer_share_mean_per_positive_person": 12059.958246993665, + "raw_employer_share_total": 893126802552.6951, + "raw_over_published_employer_mean": 1.030960994318845, + "scale_factor": 1.034460355190118, + "scale_factor_spread": 0.0, + "source_columns_missing": [], "weighted_employed_policyholders": 79790597.17620382, "weighted_employed_policyholders_by_payment": { "all": 15092018.672620341, "none": 5733391.980212707, "some": 58965186.52337077 }, + "weighted_other_policyholders": 13156453.970184255, + "weighted_other_policyholders_with_employer_share": 10825941.266066976, "weighted_policyholders": 92947051.14638808 }, "target_year": 2024 diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json index 96d8dafad2..61760666d6 100644 --- a/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json +++ b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json @@ -2,42 +2,60 @@ "anchor_gate": { "details": { "anchor": { + "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "anchor_universe_employer_total": 1046999999999.9998, + "anchor_value": 1047000000000.0, + "cross_check": { "label": "Private group health insurance", "line": 17, "published": "2026-09-30", + "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", "values": { - "2023": 923195000000.0, - "2024": 977034000000.0, - "2025": 1027929000000.0 + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 } }, - "anchor_value": 977034000000.0, - "cross_check": { - "employee_contribution": { - "2023": 357800000000.0, - "2024": 382100000000.0 - }, - "employer_contribution": { - "2023": 975700000000.0, - "2024": 1047000000000.0 - }, - "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "Table 24 Employer-Sponsored Private Health Insurance" + "cross_check_ratio": 1.0716106092520832, + "employed_share_band": [ + 0.8, + 0.95 + ], + "employed_share_of_anchor_universe": 0.8841568448185655, + "employer_premium_by_sector": { + "federal": 38124900985.39438, + "private": 713245726890.8082, + "state_and_local": 174341588648.83527 }, - "cross_check_employer_ratio": 0.9331747851002865, - "cross_check_pre_tax_ratio": 0.47395266086209464, + "employer_premium_mean_per_positive_person": 12514.898608320464, "employer_premium_positive_persons": 73968814.73011559, - "employer_premium_total": 977034000000.0, - "pre_tax_premium_total": 181097311715.40637, - "relative_error": 0.0, + "employer_premium_total": 925712216525.0378, + "other_policyholder_employer_total": 121287783474.96194, + "private_active_cross_check": { + "employer_premium_private_sector": 713245726890.8082, + "meps_ic_private_active_employer_total": 668573745863.8779, + "ratio": 1.0668168340490378 + }, + "relative_error": -2.220446049250313e-16, "relative_tolerance": 0.05, - "time_period": 2024 + "scale_factor": 1.0403508274129025, + "time_period": 2024, + "weighted_employed_policyholders": 79689927.57027404, + "weighted_other_policyholders": 12732394.803675905, + "weighted_policyholders": 92422322.37394994 }, "failures": [], "passed": true @@ -45,23 +63,20 @@ "by_vintage": { "2023": { "employer_premium_positive_persons": 24692526.63040858, - "employer_premium_total": 326526181481.6512, - "mean_per_positive_person": 13223.684492438199, - "pre_tax_premium_total": 58750093810.022896, + "employer_premium_total": 309297757165.9197, + "mean_per_positive_person": 12525.966329627154, "weighted_persons": 108650789.68666664 }, "2024": { "employer_premium_positive_persons": 24680489.296666663, - "employer_premium_total": 324810912508.5698, - "mean_per_positive_person": 13160.635050798553, - "pre_tax_premium_total": 60004130823.53, + "employer_premium_total": 307768815338.1516, + "mean_per_positive_person": 12470.126164788748, "weighted_persons": 108650789.68666667 }, "2025": { "employer_premium_positive_persons": 24595798.803040348, - "employer_premium_total": 325696906009.779, - "mean_per_positive_person": 13241.973095401916, - "pre_tax_premium_total": 62343087081.853455, + "employer_premium_total": 308645644020.96655, + "mean_per_positive_person": 12548.713969103293, "weighted_persons": 108650789.68666668 } }, @@ -92,13 +107,39 @@ } ], "issue": "PolicyEngine/microcosm#454", - "nhe_concept_check": { - "nhe_table_24_employer_contribution_2024": 1047000000000.0, - "non_employed_share_of_all": 0.11462618085949701, - "raw_all_policyholders_over_nhe": 0.9847292814494486, - "raw_all_policyholders_total": 1031011557677.5726, - "raw_employed_universe_total": 912830640398.9915, - "raw_non_employed_or_no_employer_total": 118180917278.58118 + "other_policyholder_sensitivity": { + "as_built_other_priced_as_active": { + "employer_premium_total": 925712216525.038, + "mean_per_positive_person": 12514.898608320465 + }, + "bea_anchor_all_on_employed": { + "employer_premium_total": 977034000000.0, + "mean_per_positive_person": 13208.728618470228 + }, + "bea_anchor_other_priced_as_active": { + "employer_premium_total": 863851298720.4623, + "mean_per_positive_person": 11678.587819371325 + }, + "other_65_plus_priced_at_half": { + "employer_premium_total": 948058754883.9099, + "mean_per_positive_person": 12817.006171357754 + }, + "other_not_priced_all_anchor_on_employed": { + "employer_premium_total": 1047000000000.0, + "mean_per_positive_person": 14154.61372228431 + }, + "raw_other_65_plus_share_of_other": 0.4069439742577271, + "weighted_other_policyholders_65_plus": 5730053.275654608, + "weighted_other_policyholders_by_pemlr": { + "0": 96077.96836891444, + "1": 1388904.5982447905, + "2": 77057.91946861018, + "3": 213860.3873739363, + "4": 792436.9515057954, + "5": 7140960.568448426, + "6": 942816.4562687919, + "7": 2080279.9539966392 + } }, "person_rows": 421119, "pool_census_person_columns": [ @@ -493,10 +534,10 @@ ], "puf_support_clone": { "anchor_gate": { - "employer_premium_total": 977034000000.0, + "anchor_universe_employer_total": 1046999999999.9998, + "employer_premium_total": 925712216525.0378, "failures": [], - "passed": true, - "pre_tax_premium_total": 181097311715.4063 + "passed": true }, "person_rows": 842238, "signal_gate": { @@ -504,7 +545,7 @@ "passed": true } }, - "seconds": 54.4, + "seconds": 35.1, "seed": 0, "signal_gate": { "failures": [], @@ -512,80 +553,78 @@ }, "summary": { "anchor": { + "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "anchor_universe_employer_total": 1046999999999.9998, + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "clone_disagreement_source_persons": 0, + "cross_check": { "label": "Private group health insurance", "line": 17, "published": "2026-09-30", + "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", "values": { - "2023": 923195000000.0, - "2024": 977034000000.0, - "2025": 1027929000000.0 + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 } }, - "cells_sha256": "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8", - "clone_disagreement_source_persons": 0, - "cross_check": { - "employee_contribution": { - "2023": 357800000000.0, - "2024": 382100000000.0 - }, - "employer_contribution": { - "2023": 975700000000.0, - "2024": 1047000000000.0 - }, - "relation": "upper bound: NHEA counts employer contributions for active employees, COBRA enrollees and retirees", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "Table 24 Employer-Sponsored Private Health Insurance" - }, + "employed_share_of_anchor_universe": 0.8841568448185655, "employer_premium_by_sector": { - "federal": 39589380243.32626, - "private": 756704453166.5498, - "state_and_local": 180740166590.12393 + "federal": 38124900985.39438, + "private": 713245726890.8082, + "state_and_local": 174341588648.83527 }, "employer_premium_by_tier": { - "employee_plus_one": 208020070710.58252, - "family": 414697085204.7131, - "single": 354316844084.70447 + "employee_plus_one": 198151264745.7921, + "family": 393140740531.96185, + "single": 334420211247.28394 }, - "employer_premium_mean_per_positive_person": 13208.728618470228, + "employer_premium_mean_per_positive_person": 12514.898608320464, "employer_premium_outside_universe_rows": 0, "employer_premium_positive_persons": 73968814.73011559, "employer_premium_positive_share": 0.2269313609943412, - "employer_premium_total": 977034000000.0, + "employer_premium_total": 925712216525.0378, "employer_premium_where_employer_pays_none_rows": 0, "employer_premium_without_raw_share_rows": 0, + "meps_ic_private_active_employer_total": { + "2024": 668573745863.8779, + "2025": 714391090330.8774 + }, "negative_rows": 0, "nonfinite_rows": 0, - "pre_tax_eligible_weight": 60693311.22982113, - "pre_tax_ineligible_rows": 0, - "pre_tax_not_reported_premium_rows": 0, - "pre_tax_premium_positive_persons": 52487359.3024551, - "pre_tax_premium_positive_share": 0.161027696941799, - "pre_tax_premium_total": 181097311715.40637, - "pre_tax_selected_share_of_eligible": 0.8647964370193382, - "pre_tax_shares": { - "50plus": 0.9261603375527426, - "lt50": 0.4953271028037383, - "total": 0.7215447154471545 - }, + "other_policyholder_employer_total": 121287783474.96194, + "raw_anchor_universe_total": 1006391279183.8038, "raw_employer_share_band": [ 6000.0, 20000.0 ], - "raw_employer_share_mean_per_positive_person": 12340.74986505553, - "raw_employer_share_total": 912830640398.9915, - "scale_factor": 1.0703343607889255, - "scale_factor_spread": 4.440892098500626e-16, + "raw_employer_share_mean_per_positive_person": 12029.498394730888, + "raw_employer_share_total": 889807738056.072, + "raw_over_published_employer_mean": 1.0307192789751904, + "scale_factor": 1.0403508274129025, + "scale_factor_spread": 0.0, + "source_columns_missing": [], "weighted_employed_policyholders": 79689927.57027404, "weighted_employed_policyholders_by_payment": { "all": 14922582.431638531, "none": 5721112.840158437, "some": 59046232.29847707 }, + "weighted_other_policyholders": 12732394.803675905, + "weighted_other_policyholders_with_employer_share": 10538098.517560685, "weighted_policyholders": 92422322.37394994 }, "target_year": 2024 diff --git a/experiments/us-esi-454/run_stage_on_pinned_pool.py b/experiments/us-esi-454/run_stage_on_pinned_pool.py index 64cec8a6a1..ce95fa8f5c 100644 --- a/experiments/us-esi-454/run_stage_on_pinned_pool.py +++ b/experiments/us-esi-454/run_stage_on_pinned_pool.py @@ -18,7 +18,6 @@ from __future__ import annotations import argparse -import dataclasses import hashlib import json import time @@ -34,7 +33,6 @@ from microcosm.build.us_runtime.asec_sources import ASEC_SOURCE_ARTIFACTS from microcosm.build.us_runtime.esi_premiums import ( US_ESI_EMPLOYER_PREMIUM_COLUMN, - US_ESI_PRE_TAX_PREMIUM_COLUMN, us_esi_premiums_anchor_gate, us_esi_premiums_signal_gate, us_esi_premiums_summary, @@ -68,7 +66,6 @@ def _by_vintage(frame, years) -> dict[str, dict[str, float]]: for year in years: mask = person["source_year"].to_numpy() == year employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy() - pre_tax = person[US_ESI_PRE_TAX_PREMIUM_COLUMN].to_numpy() holders = mask & (employer > 0) out[str(year)] = { "weighted_persons": float(weights[mask].sum()), @@ -77,43 +74,56 @@ def _by_vintage(frame, years) -> dict[str, dict[str, float]]: "mean_per_positive_person": float( weights[holders] @ employer[holders] / weights[holders].sum() ), - "pre_tax_premium_total": float(weights[mask] @ pre_tax[mask]), } return out -def _all_policyholder_concept_check(frame) -> dict[str, float]: - """Raw MEPS-IC shares over EVERY current policyholder (NHE's concept). +def _other_policyholder_sensitivity(frame, target_year: int) -> dict[str, object]: + """How the employed column moves with the price of non-employed coverage. - Non-employed policyholders (retirees, COBRA, others) take their State's - all-sizes private cell; this is diagnostics only, never the column. + No MEPS-IC table prices retiree coverage, so the stage prices every + policyholder outside the column as an active employee. This re-scales + the anchor universe under two alternatives: the 65-and-over policyholders + outside the column priced at half the active cell, and nobody outside the + column priced at all (the whole anchor loaded onto employed workers). It + also restates the as-built column and the all-on-employed reading against + BEA NIPA 7.8 line 17, the second estimate of the anchor's concept. """ person = esi._person_with_state(frame) - codes = esi._esi_person_codes(person) - holder = codes.owner == 1 - widened = dataclasses.replace( - codes, - employed=np.ones(len(codes.owner), dtype=bool), - sector=np.where(np.isin(codes.sector, (1, 2, 3, 4, 5, 6)), codes.sector, 4), - noemp=np.where(codes.employed, codes.noemp, 0), - ) - premium, contribution = esi._cell_values( - widened, esi.load_meps_ic_esi_premium_cells() - ) - raw = esi._raw_employer_share(widened, premium, contribution) + codes, raw, _premium, _contribution = esi._person_raw_shares(person) weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - employed = codes.universe - total = float(weights[holder] @ raw[holder]) - employed_total = float(weights[employed] @ raw[employed]) - nhe = float(esi.EMPLOYER_PREMIUM_CROSS_CHECK["employer_contribution"]["2024"]) + universe = codes.universe + other = codes.policyholder & ~universe + age = person["A_AGE"].to_numpy(dtype=np.float64) + older = other & (age >= 65) + anchor = float(esi.EMPLOYER_PREMIUM_ANCHOR["values"][str(target_year)]) + bea = float(esi.EMPLOYER_PREMIUM_CROSS_CHECK["values"][str(target_year)]) + employed = float(weights[universe] @ raw[universe]) + everyone = float(weights @ raw) + older_raw = float(weights[older] @ raw[older]) + positive = float(weights[universe & (raw > 0)].sum()) + pemlr = person["PEMLR"].to_numpy(dtype=np.int64) + + def column(total_raw: float, total_anchor: float = anchor) -> dict[str, float]: + total = total_anchor * employed / total_raw + return { + "employer_premium_total": total, + "mean_per_positive_person": total / positive, + } + return { - "raw_all_policyholders_total": total, - "raw_employed_universe_total": employed_total, - "raw_non_employed_or_no_employer_total": total - employed_total, - "non_employed_share_of_all": (total - employed_total) / total, - "nhe_table_24_employer_contribution_2024": nhe, - "raw_all_policyholders_over_nhe": total / nhe, + "as_built_other_priced_as_active": column(everyone), + "other_65_plus_priced_at_half": column(everyone - 0.5 * older_raw), + "other_not_priced_all_anchor_on_employed": column(employed), + "bea_anchor_other_priced_as_active": column(everyone, bea), + "bea_anchor_all_on_employed": column(employed, bea), + "weighted_other_policyholders_by_pemlr": { + str(code): float(weights[other & (pemlr == code)].sum()) + for code in sorted(set(pemlr[other].tolist())) + }, + "weighted_other_policyholders_65_plus": float(weights[older].sum()), + "raw_other_65_plus_share_of_other": older_raw / (everyone - employed), } @@ -198,7 +208,9 @@ def main() -> None: ], "summary": _jsonable(us_esi_premiums_summary(staged)), "by_vintage": _by_vintage(staged, years), - "nhe_concept_check": _all_policyholder_concept_check(staged), + "other_policyholder_sensitivity": _other_policyholder_sensitivity( + staged, args.target_year + ), "signal_gate": {"passed": signal.passed, "failures": list(signal.failures)}, "anchor_gate": { "passed": anchor.passed, @@ -213,12 +225,13 @@ def main() -> None: "anchor_gate": { "passed": cloned_anchor.passed, "failures": list(cloned_anchor.failures), - "employer_premium_total": cloned_anchor.details.get( - "employer_premium_total" - ), - "pre_tax_premium_total": cloned_anchor.details.get( - "pre_tax_premium_total" - ), + **{ + key: cloned_anchor.details.get(key) + for key in ( + "employer_premium_total", + "anchor_universe_employer_total", + ) + }, }, "person_rows": len(cloned.table("person")), }, diff --git a/packages/microcosm-build/src/microcosm/build/source_manifest.py b/packages/microcosm-build/src/microcosm/build/source_manifest.py index 86850f5fc5..f1a15a61a7 100644 --- a/packages/microcosm-build/src/microcosm/build/source_manifest.py +++ b/packages/microcosm-build/src/microcosm/build/source_manifest.py @@ -93,7 +93,6 @@ "derive_immigration_status", "derive_medicare_take_up", "derive_other_health_insurance_premiums", - "derive_pre_tax_health_insurance_premiums", "derive_prior_year_income", "derive_snap_abawd_discretionary_exemption", "derive_snap_take_up", diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index 7086d370ab..ba54255837 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_452 -EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_800 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_252 +EXPECTED_AUTHORED_FIELD_COUNT = 32_436 +EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_216 class FieldUsageError(AssertionError): @@ -453,12 +453,12 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "6a781915fd491d2c4b16d2b7d482f69cf362c904130093c59f9629f7a319269b", ), "resolved_seed_protocol": ( - 843, - "c51302fee9ab90cd34097f3d6cc8a765f2a12cb236e68a5772b87d0756f45242", + 826, + "7ff2d5d1c2fd8026d17a57244f969dc0e9625a9e47304b693ad15df329041282", ), "resolved_seed_site_bindings": ( - 280, - "1f88dfbe50cabf8e57c36f717b0cc952b5c423c1c2d62ac4219ec3c3872ead9f", + 277, + "368fc19a8f07c4abf38ebcf2fbc4414d33c403c0f1f9881c2d2e3f5d6160feb6", ), "resolved_vintage_authorities": ( 63, @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_761, - "bdecb4db8c7c0a6b1dac1b39b4490007627f83bee305abcc1456304fdbfb5258", + 1_748, + "dcc891c12eb675ee7f21c903995db98c3b8c6ccba569d71f0fd59b589abce04c", ), "spine_assembly_household_mass_shares": ( 2, @@ -513,8 +513,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "29c6c1b3243e178783e7ab139993ba3a9b42d62edd4e9e4e1f3b28688daf2c6d", ), "spine_seed_site_bindings": ( - 280, - "6cb87c6d4e3cca9907cc85d78a0ebd391281dc70391d30111d57a8bc84848bb6", + 277, + "dfa7ae701f62f7b1f06954798d29cfba3fc53a0be21e1751646d1f5bafd6071f", ), "spine_support_roles": ( 29, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 0ac360e9e9..f82307a13f 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -294,7 +294,6 @@ "source_joint_count_calibration", "snap_take_up_assignment", "pregnancy_assignment", - "pre_tax_health_premium_assignment", "wic_claim_assignment", "snap_discretionary_exemption_assignment", "immigration_ead_workers_assignment", @@ -351,18 +350,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "69942294839a762738095894b87f6e72ea4faac03ebc020a4406c4f27bbb5c40", + "full_checkpoint": "e2a40490a6b705bab15bf2e7dff8eea4d7841ad70d3b7858874778d4e4829021", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "b4f9cbbee81eff7f59304a114ed105910140f0799797c87f26ae9eb5090060f8", + "late_resource_semantics": "3bfce32d85079804f869686bd6bf983dbbb7e262c05a982a9d40d85ae0f69238", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", - "seed_map": "bdb924d0b21e55e387a533d179dc6c0f7d239bee79bd9de640729af17c9b2301", - "seed_protocol": "9c2eb0d42f0ebd7dab6f0aebad37b3bc6454ec6b3235df0a489e361d3a4f6b8d", - "source_manifest": "b7f2650fd828f593bb53c8f6d37cbf1e0318111b8a81d4bc279954ed25e33fcd", + "seed_map": "5641418e4a815da24744d59dfcf82fb11aa2f2ddc290ffcab9627383ea6fa682", + "seed_protocol": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb", + "source_manifest": "b90f0b264f9357eeda5169416babb50f3f360927a70aa0fd6ec78aa54af50427", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } @@ -435,9 +434,9 @@ "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 113, - "seed_owner_rows": 55, - "seed_sites": 54, + "seed_owner_bindings": 112, + "seed_owner_rows": 54, + "seed_sites": 53, "seed_streams": 14, "source_operators": 16, "source_stages": 38, @@ -1779,7 +1778,7 @@ def add( "seed_site_definitions_exact", clauses={ "seed site definitions differ": site_definitions_exact, - "seed site count differs": len(protocol_sites) == 54, + "seed site count differs": len(protocol_sites) == 53, }, homes=("/bundle/seed_protocol",), consumers=("compiler_ir.seed_stream_map.sites", "compiler_ir.node_slices"), @@ -1787,7 +1786,7 @@ def add( "sites": len(protocol_sites), "sha256": sha256_json([site.to_wire() for site in protocol.sites]), }, - expected={"sites": 54, "relation": "all site fields preserved exactly"}, + expected={"sites": 53, "relation": "all site fields preserved exactly"}, ) binding_by_site = {binding.site: binding for binding in spec.seed_site_bindings} binding_exact = set(binding_by_site) == set(compiled_sites) and all( @@ -1802,7 +1801,7 @@ def add( "owner binding count differs": sum( len(site.owners) for site in compiled_sites.values() ) - == 113, + == 112, "one or more sites have no owner": all( site.owners for site in compiled_sites.values() ), @@ -1812,7 +1811,7 @@ def add( observed={ "bindings": sum(len(site.owners) for site in compiled_sites.values()) }, - expected={"bindings": 113, "coverage": "all 54 sites"}, + expected={"bindings": 112, "coverage": "all 53 sites"}, ) expected_owner_sites: dict[tuple[str, str], list[str]] = {} for site in protocol.sites: @@ -1836,13 +1835,13 @@ def add( "seed owner rows differ": _json_equal( compiled_owner_rows, expected_owner_rows ), - "seed owner row count differs": len(compiled_owner_rows) == 55, + "seed owner row count differs": len(compiled_owner_rows) == 54, }, homes=("/spine/seed_site_bindings", "/bundle/seed_protocol"), consumers=("compiler_ir.seed_stream_map.owners",), observed={"owner_rows": len(compiled_owner_rows)}, expected={ - "owner_rows": 55, + "owner_rows": 54, "relation": "site-order-preserving owner aggregation", }, ) @@ -1856,7 +1855,7 @@ def add( "seed groups overlap": groups_disjoint, "seed groups do not cover the protocol exactly": set(grouped_ids) == set(protocol_sites), - "seed group cardinality differs": len(grouped_ids) == 54, + "seed group cardinality differs": len(grouped_ids) == 53, }, homes=("/bundle/seed_protocol", "/spine/seed_site_bindings"), consumers=("compiler_ir.seed_stream_map",), @@ -1866,7 +1865,7 @@ def add( }, expected={ "groups": len(EXPECTED_SEED_GROUPS), - "sites": 54, + "sites": 53, "partition": "disjoint and exhaustive", }, ) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json index 6521834923..b1a2f51286 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/schema/sources.schema.json @@ -1722,7 +1722,7 @@ "type": "object", "additionalProperties": false, "properties": { - "series": { + "row": { "type": "string" }, "sha256": { @@ -1732,6 +1732,9 @@ "type": "string", "x-spec-surface": "operational" }, + "table": { + "type": "string" + }, "values": { "type": "object", "additionalProperties": { @@ -1740,12 +1743,17 @@ } }, "required": [ - "series", + "row", "sha256", "source", + "table", "values" ] }, + "anchor_universe": { + "type": "string", + "x-spec-surface": "documentation" + }, "cells_resource": { "type": "string" }, @@ -1759,6 +1767,14 @@ "kind": { "const": "derive_employer_sponsored_insurance_premiums" }, + "no_contribution_share": { + "type": "string", + "x-spec-surface": "documentation" + }, + "other_policyholder_cell": { + "type": "string", + "x-spec-surface": "documentation" + }, "payment_column": { "type": "string" }, @@ -1766,6 +1782,10 @@ "type": "string", "x-spec-surface": "documentation" }, + "scaling": { + "type": "string", + "x-spec-surface": "documentation" + }, "sector_column": { "type": "string" }, @@ -1786,12 +1806,16 @@ }, "required": [ "anchor", + "anchor_universe", "cells_resource", "cells_sha256", "government_cell_aging", "kind", + "no_contribution_share", + "other_policyholder_cell", "payment_column", "payment_rule", + "scaling", "sector_column", "size_column", "suppressed_cell_fallback", @@ -1799,42 +1823,6 @@ "universe" ] }, - { - "type": "object", - "additionalProperties": false, - "properties": { - "employer_payment_codes": { - "type": "array", - "items": { - "type": "integer" - } - }, - "kind": { - "const": "derive_pre_tax_health_insurance_premiums" - }, - "pre_tax_share": { - "type": "string", - "x-spec-surface": "documentation" - }, - "premium_column": { - "type": "string" - }, - "seed_from_build_config": { - "type": "boolean" - }, - "wage_column": { - "type": "string" - } - }, - "required": [ - "employer_payment_codes", - "kind", - "pre_tax_share", - "premium_column", - "seed_from_build_config", - "wage_column" - ] - }, { "type": "object", "additionalProperties": false, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py index 1151c3018f..e837a009e5 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/seeds.py @@ -315,7 +315,6 @@ def _site( "microcosm.build.us_runtime.congressional_district_geography", "microcosm.build.us_runtime.disability_benefits", "microcosm.build.us_runtime.energy_subsidy", - "microcosm.build.us_runtime.esi_premiums", "microcosm.build.us_runtime.geography_ladder", "microcosm.build.us_runtime.housing_inputs", "microcosm.build.us_runtime.immigration", @@ -725,15 +724,6 @@ def _stable_site( ), candidate_universe="all_person_rows_before_sex_and_age_eligibility_mask", ), - _stable_site( - "pre_tax_health_premium_assignment", - salt="pre_tax_health_insurance_premiums", - key_grammar=( - "source_year:source_household_id:source_person_id_if_complete", - "else_person_id", - ), - candidate_universe="all_person_rows_before_pre_tax_eligibility_mask", - ), _stable_site( "wic_claim_assignment", salt="would_claim_wic", diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 82906e87ba..0d55025c85 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -72,7 +72,7 @@ "status": "required" }, "employer_sponsored_insurance_premiums": { - "note": "Employer share of the premium for employed current ESI policyholders (PolicyEngine/microcosm#454), written by the meps_esi_premiums base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person files) and MEPS-IC tier x firm-size x State cells, scaled to BEA NIPA 7.8 line 17. A release must persist it: CBO household market income (gov.household.cbo_market_income_additions) and every ESI-to-wages reform bind through it. Required with NO reviewed exclusion; absent on pre-#454 bases by construction — the intended red gate until a base built with the stage ships. The release also holds its total to the anchor (us_esi_premiums_anchor_gate).", + "note": "Employer share of the premium for employed current ESI policyholders (PolicyEngine/microcosm#454), written by the meps_esi_premiums base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person files) and MEPS-IC tier x firm-size x State cells, scaled over every policyholder to CMS NHE Table 24. A release must persist it: CBO household market income (gov.household.cbo_market_income_additions) and every ESI-to-wages reform bind through it. Required with NO reviewed exclusion; absent on pre-#454 bases by construction — the intended red gate until a base built with the stage ships. The release also holds the anchor-universe total to the anchor (us_esi_premiums_anchor_gate).", "status": "required" }, "employment_income_before_lsr": { @@ -308,10 +308,6 @@ "pre_subsidy_rent": { "status": "required" }, - "pre_tax_health_insurance_premiums": { - "note": "Employee premium paid through section 125 payroll deduction (PolicyEngine/microcosm#454), written by the same meps_esi_premiums stage: an employed policyholder with wages, a reported premium (PHIP_VAL) and an employee share pays it pre-tax at the MEPS-IC 2025 firm-size pretax-offer rate. PolicyEngine-US 2.2.1 subtracts it from income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, fica_pre_tax_contributions); absent, both wage bases overstate. Required with NO reviewed exclusion; absent on pre-#454 bases by construction.", - "status": "required" - }, "previous_year_income_available": { "status": "required" }, @@ -553,11 +549,11 @@ } }, "counts": { - "required": 169, + "required": 168, "reviewed_exclusion": 6, - "total": 175 + "total": 174 }, - "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893), and the employee section 125 premium leaf pre_tax_health_insurance_premiums (the meps_esi_premiums base-builder stage; PolicyEngine/microcosm#454). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", + "derivation": "Required surface = input columns in the pinned, sha-verified ecps_parity_reference.json populated layers, plus the documented post-reference fsla_overtime_premium, qualified_passenger_vehicle_loan_interest, five desired retirement-contribution inputs, meets_ssi_disability_criteria required by shipped validation probes, the #282 Schedule-D capital-gain-distributions route leg schedule_d_capital_gain_distributions (PolicyEngine/microcosm#462), the three ASEC reported-receipt inputs receives_wic, receives_snap, receives_tanf that the ACS local-area transfer requires in its donor (PolicyEngine/microcosm#978 option 1), and the engine's declared dataset source input is_spm_independent_minor_role (the spm_independence_role base-builder stage; PolicyEngine/microcosm#893). status='reviewed_exclusion' for ecps_parity_known_gaps.json entries (reason+issue from that register); EXCEPT every primary-source restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS (including the Section 199A QBI family), and the SSI countable-resource asset inputs (bank_account_assets, stock_assets, bond_assets), which are status='required' with NO exclusion per PolicyEngine/microcosm#368 so the gate fails on today's artifacts and asset restoration (Deliverable 2) turns it green. All other populated layers are 'required'. Regenerate with tools/build_us_release_input_coverage_manifest.py.", "description": "Declared full-coverage contract for a US release: every input column the reference eCPS exports must be persisted as a key with non-default signal, or carry a reviewed exclusion. Enforced as a hard release gate (microcosm.build.us_runtime.release_input_coverage) that generalizes assert_required_us_release_source_columns from 5 columns to the full eCPS input surface.", "issue": "PolicyEngine/microcosm#368", "reference": { @@ -584,23 +580,7 @@ "neutralized_variable": "employer_sponsored_insurance_premiums", "parameter_changes": {}, "period": 2024, - "reason": "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums to CBO household market income through gov.household.cbo_market_income_additions, its only consumer, so neutralizing it lowers cbo_household_market_income by the weighted employer-premium total: about $977 billion at 2024 (BEA NIPA 7.8 line 17, the meps_esi_premiums anchor; the stage measured $977.0 billion over 74.0 million weighted employed policyholders on the default 2023-2025 pool, before calibration). The floor sits at about half of that. A structural zero means the column was dropped or the stage never ran." - }, - { - "binding_inputs": [ - "pre_tax_health_insurance_premiums" - ], - "budget_measure": "income_tax", - "effect_direction": "baseline_minus_reform", - "expected_sign": "negative", - "id": "pre_tax_health_insurance_premium_neutralization", - "issue": "PolicyEngine/microcosm#454", - "min_abs_effect": 5000000000.0, - "name": "Pre-tax health insurance premium neutralization", - "neutralized_variable": "pre_tax_health_insurance_premiums", - "parameter_changes": {}, - "period": 2024, - "reason": "PolicyEngine-US 2.2.1 subtracts pre_tax_health_insurance_premiums from employment_income in irs_employment_income (through gov.irs.gross_income.pre_tax_contributions), so neutralizing it raises taxable wages and income tax: baseline-minus-reform is negative. The meps_esi_premiums stage measured $181.1 billion of pre-tax premiums over 52.5 million weighted workers on the default 2023-2025 pool; at a typical 10-22 percent marginal rate that is roughly $18-40 billion, so a $5 billion floor separates a structural zero (the column dropped) from any plausible effect." + "reason": "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums to CBO household market income through gov.household.cbo_market_income_additions, its only consumer, so neutralizing it lowers cbo_household_market_income by the weighted employer-premium total: about $926 billion at 2024 (the employed policyholders' part of CMS NHE Table 24's $1,047.0 billion, the meps_esi_premiums anchor; the stage measured $925.7 billion over 74.0 million weighted employed policyholders on the default 2023-2025 pool, before calibration). The floor sits at about half of that. A structural zero means the column was dropped or the stage never ran." }, { "binding_inputs": [ diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index e2bfce79b8..202943187a 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1754,7 +1754,7 @@ "kind": "public_microdata", "format": "census_asec", "vintage": "pinned Census ASEC person members, income years 2022-2025", - "locator": "Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM join; every pinned H5 lacks them); PEIO1COW, PHIP_VAL and WSAL_VAL from the pinned H5 inputs; household GESTFIPS as state_fips", + "locator": "Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM join; every pinned H5 lacks them); PEIO1COW from the pinned H5 inputs; household GESTFIPS as state_fips", "source_columns": [ "NOW_OWNGRP", "NOW_HIPAID", @@ -1763,8 +1763,6 @@ "PEMLR", "NOEMP", "PEIO1COW", - "PHIP_VAL", - "WSAL_VAL", "state_fips" ] }, @@ -1775,7 +1773,7 @@ "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2025/ic25_iia_f.pdf", "sha256": "ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", "size_bytes": 12530005, - "publication": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee) and II.C/D/E.2 (average employee contribution)" + "publication": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee), II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent of enrollees whose coverage required no employee contribution), and the enrollment rows II.B.1, II.B.2, II.B.2.b and II.C/D/E.4" }, { "kind": "administrative_table", @@ -1784,7 +1782,7 @@ "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf", "sha256": "4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", "size_bytes": 10687163, - "publication": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025" + "publication": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025 and carry the 2024 private-sector enrollment" }, { "kind": "administrative_table", @@ -1797,12 +1795,13 @@ }, { "kind": "administrative_table", - "format": "pdf", - "vintage": "2025", - "locator": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_1/2025/ic25_ia_g.pdf", - "sha256": "cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3", - "size_bytes": 9210366, - "publication": "MEPS-IC 2025 Series I (private sector, national): Table I.A.2 (percent of establishments that offer health insurance) and Table I.A.2.j (percent of establishments that offer pretax employee contributions to health insurance), by firm size" + "format": "xlsx", + "vintage": "NHE 2024 release (data through CY2024)", + "locator": "https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "size_bytes": 520391, + "measure": "NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024", + "publication": "scale anchor: employer contributions for active employees, COBRA enrollees and retirees of private, federal and State and local employers" }, { "kind": "administrative_table", @@ -1812,14 +1811,7 @@ "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "size_bytes": 944273, "measure": "NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025", - "publication": "scale anchor: employer contributions for group health insurance as a supplement to wages and salaries" - }, - { - "kind": "administrative_validation_only", - "format": "xlsx", - "vintage": "NHE 2024 release (data through CY2024)", - "locator": "https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx", - "measure": "employer and employee contributions to employer-sponsored private health insurance premiums (upper bounds: NHEA includes COBRA and retiree enrollees)" + "publication": "cross-check, not gated: a second estimate of employer contributions for active and retired employees, recorded beside the anchor verdict" } ], "operations": [ @@ -1831,47 +1823,38 @@ { "kind": "derive_employer_sponsored_insurance_premiums", "cells_resource": "microcosm.build.us_runtime.data/meps_ic_esi_premium_cells.json", - "cells_sha256": "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8", + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "anchor_universe": "NOW_OWNGRP == 1", "universe": "NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6)", "tier_column": "NOW_GRPFTYP2", "size_column": "NOEMP", "sector_column": "PEIO1COW", "payment_column": "NOW_HIPAID", - "payment_rule": "1 (employer paid all): average total premium; 2 (some): average total premium minus average employee contribution; 3 (none): 0", + "payment_rule": "1 (employer paid all): average total premium; 2 (some): average total premium minus average employee contribution / (1 - no-contribution share); 3 (none): 0", + "no_contribution_share": "MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; government cells take the 50-or-more share", + "other_policyholder_cell": "PEIO1COW outside 1-6: the State's all-sizes private cell", "suppressed_cell_fallback": "national firm-size cell times the State total-to-national ratio", "government_cell_aging": "MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same tier and measure", + "scaling": "one factor, anchor / weighted raw share of the anchor universe; written to the universe only", "anchor": { - "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", - "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", - "series": "B4923C", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "row": "Employer Contribution to Private Health Insurance Premiums", "values": { - "2023": 923195000000, - "2024": 977034000000, - "2025": 1027929000000 + "2023": 975700000000, + "2024": 1047000000000 } } - }, - { - "kind": "derive_pre_tax_health_insurance_premiums", - "seed_from_build_config": true, - "premium_column": "PHIP_VAL", - "wage_column": "WSAL_VAL", - "employer_payment_codes": [ - 2, - 3 - ], - "pre_tax_share": "MEPS-IC 2025 Table I.A.2.j pretax-contribution offer percent / Table I.A.2 health-insurance offer percent, by firm size" } ], "outputs": [ - "employer_sponsored_insurance_premiums", - "pre_tax_health_insurance_premiums" + "employer_sponsored_insurance_premiums" ], "nonnegative_outputs": [ - "employer_sponsored_insurance_premiums", - "pre_tax_health_insurance_premiums" + "employer_sponsored_insurance_premiums" ], - "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Universe: employed (PEMLR 1-2) current ESI policyholders (NOW_OWNGRP 1) whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders (retiree and COBRA coverage is not compensation of a current job), the unincorporated self-employed and workers without pay carry 0. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium; some (2): premium minus contribution; none (3): 0. One factor scales every share so the weighted total equals BEA NIPA 7.8 line 17 for the build year ($977.0B CY2024 in the 2026-09-30 vintage; the retired us-data target was the earlier $1,002.9B vintage). NHE Table 24 ($1,047.0B CY2024, banked as cms_nhe.cy2024.esi_employer_contribution_premiums) counts COBRA and retiree enrollees and is an upper bound; applying the same cells to every current policyholder reproduces it within 2 percent on both pinned pools ($1,031.0B on income years 2023-2025, $1,036.8B on 2022-2024), non-employed and no-employer policyholders carrying 11.5 percent (experiments/us-esi-454). pre_tax_health_insurance_premiums: an employed policyholder with wages (WSAL_VAL > 0), a reported premium (PHIP_VAL > 0) and an employee share (NOW_HIPAID 2-3) pays PHIP_VAL through payroll with the firm-size probability that an insurance-offering employer offers pretax contributions (MEPS-IC 2025 Table I.A.2.j / Table I.A.2: 49.5 percent under 50, 92.6 percent 50+, 72.2 percent all sizes); establishment-weighted, so a lower estimate. Seeded blake2b draws keyed by source identity; support clones agree. PolicyEngine-US 2.2.1 subtracts it from income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, fica_pre_tax_contributions). Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples) and us_esi_premiums_anchor_gate (calibrated total within 5 percent of BEA, at most NHE)." + "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active employees, COBRA enrollees and retirees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals CMS NHE Table 24's employer contribution for the build year ($1,047.0B CY2024, banked as cms_nhe.cy2024.esi_employer_contribution_premiums), the anchor microcosm#454 chose; the column carries the employed policyholders' part. BEA NIPA 7.8 line 17 ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of NHE Table 24; employed share within its band)." }, { "stage": "prior_year_income", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 6877c0b5a7..b9b0500e0e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91 + sha256: e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe stage_manifest: version: 1 country: us @@ -1792,8 +1792,7 @@ stages: vintage: pinned Census ASEC person members, income years 2022-2025 locator: Census CPS ASEC person file NOW_OWNGRP, NOW_HIPAID, NOW_GRPFTYP, NOW_GRPFTYP2, PEMLR and NOEMP, restored from the pinned official archives by asec_census_person_columns (exact PERIDNUM - join; every pinned H5 lacks them); PEIO1COW, PHIP_VAL and WSAL_VAL from the pinned H5 inputs; household - GESTFIPS as state_fips + join; every pinned H5 lacks them); PEIO1COW from the pinned H5 inputs; household GESTFIPS as state_fips source_columns: - NOW_OWNGRP - NOW_HIPAID @@ -1802,8 +1801,6 @@ stages: - PEMLR - NOEMP - PEIO1COW - - PHIP_VAL - - WSAL_VAL - state_fips - kind: administrative_table format: pdf @@ -1812,7 +1809,9 @@ stages: sha256: ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330 size_bytes: 12530005 publication: 'MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average - total premium per enrolled employee) and II.C/D/E.2 (average employee contribution)' + total premium per enrolled employee), II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent + of enrollees whose coverage required no employee contribution), and the enrollment rows II.B.1, + II.B.2, II.B.2.b and II.C/D/E.4' - kind: administrative_table format: pdf vintage: '2024' @@ -1820,7 +1819,7 @@ stages: sha256: 4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b size_bytes: 10687163 publication: MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows - age the 2024 State and local government cells to 2025 + age the 2024 State and local government cells to 2025 and carry the 2024 private-sector enrollment - kind: administrative_table format: pdf vintage: '2024' @@ -1831,14 +1830,14 @@ stages: census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released).' - kind: administrative_table - format: pdf - vintage: '2025' - locator: https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_1/2025/ic25_ia_g.pdf - sha256: cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3 - size_bytes: 9210366 - publication: 'MEPS-IC 2025 Series I (private sector, national): Table I.A.2 (percent of establishments - that offer health insurance) and Table I.A.2.j (percent of establishments that offer pretax employee - contributions to health insurance), by firm size' + format: xlsx + vintage: NHE 2024 release (data through CY2024) + locator: https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx + sha256: a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420 + size_bytes: 520391 + measure: NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024 + publication: 'scale anchor: employer contributions for active employees, COBRA enrollees and retirees + of private, federal and State and local employers' - kind: administrative_table format: xlsx vintage: BEA annual update published 2026-09-30 @@ -1846,81 +1845,75 @@ stages: sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd size_bytes: 944273 measure: NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025 - publication: 'scale anchor: employer contributions for group health insurance as a supplement to wages - and salaries' - - kind: administrative_validation_only - format: xlsx - vintage: NHE 2024 release (data through CY2024) - locator: https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx - measure: 'employer and employee contributions to employer-sponsored private health insurance premiums - (upper bounds: NHEA includes COBRA and retiree enrollees)' + publication: 'cross-check, not gated: a second estimate of employer contributions for active and retired + employees, recorded beside the anchor verdict' operations: - kind: read_table table: person weight: person_weight - kind: derive_employer_sponsored_insurance_premiums cells_resource: microcosm.build.us_runtime.data/meps_ic_esi_premium_cells.json - cells_sha256: fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8 + cells_sha256: 29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93 + anchor_universe: NOW_OWNGRP == 1 universe: NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6) tier_column: NOW_GRPFTYP2 size_column: NOEMP sector_column: PEIO1COW payment_column: NOW_HIPAID payment_rule: '1 (employer paid all): average total premium; 2 (some): average total premium minus - average employee contribution; 3 (none): 0' + average employee contribution / (1 - no-contribution share); 3 (none): 0' + no_contribution_share: MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; government + cells take the 50-or-more share + other_policyholder_cell: 'PEIO1COW outside 1-6: the State''s all-sizes private cell' suppressed_cell_fallback: national firm-size cell times the State total-to-national ratio government_cell_aging: MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same tier and measure + scaling: one factor, anchor / weighted raw share of the anchor universe; written to the universe only anchor: - source: https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx - sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd - series: B4923C + source: https://www.cms.gov/files/zip/nhe-tables.zip + sha256: a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420 + table: NHE Table 24 Employer-Sponsored Private Health Insurance + row: Employer Contribution to Private Health Insurance Premiums values: - '2023': 923195000000 - '2024': 977034000000 - '2025': 1027929000000 - - kind: derive_pre_tax_health_insurance_premiums - seed_from_build_config: true - premium_column: PHIP_VAL - wage_column: WSAL_VAL - employer_payment_codes: - - 2 - - 3 - pre_tax_share: MEPS-IC 2025 Table I.A.2.j pretax-contribution offer percent / Table I.A.2 health-insurance - offer percent, by firm size + '2023': 975700000000 + '2024': 1047000000000 outputs: - employer_sponsored_insurance_premiums - - pre_tax_health_insurance_premiums nonnegative_outputs: - employer_sponsored_insurance_premiums - - pre_tax_health_insurance_premiums notes: 'Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) - from source fields the hermetic H5 inputs lacked (microcosm#454). Universe: employed (PEMLR 1-2) current - ESI policyholders (NOW_OWNGRP 1) whose current job has an employer (PEIO1COW 1-6); dependents, non-employed - policyholders (retiree and COBRA coverage is not compensation of a current job), the unincorporated - self-employed and workers without pay carry 0. Cells: MEPS-IC 2025 Series II private-sector average - total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under - 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and - coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); - State government and local/federal employers take the MEPS-IC 2024 Series III State-government and - all-government census-division cells (latest published), aged to 2025 by the private national 2025/2024 - ratio of the same tier and measure. AHRQ suppressed nine small-firm contribution cells; each takes - the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID - 1): premium; some (2): premium minus contribution; none (3): 0. One factor scales every share so the - weighted total equals BEA NIPA 7.8 line 17 for the build year ($977.0B CY2024 in the 2026-09-30 vintage; - the retired us-data target was the earlier $1,002.9B vintage). NHE Table 24 ($1,047.0B CY2024, banked - as cms_nhe.cy2024.esi_employer_contribution_premiums) counts COBRA and retiree enrollees and is an - upper bound; applying the same cells to every current policyholder reproduces it within 2 percent - on both pinned pools ($1,031.0B on income years 2023-2025, $1,036.8B on 2022-2024), non-employed and - no-employer policyholders carrying 11.5 percent (experiments/us-esi-454). pre_tax_health_insurance_premiums: - an employed policyholder with wages (WSAL_VAL > 0), a reported premium (PHIP_VAL > 0) and an employee - share (NOW_HIPAID 2-3) pays PHIP_VAL through payroll with the firm-size probability that an insurance-offering - employer offers pretax contributions (MEPS-IC 2025 Table I.A.2.j / Table I.A.2: 49.5 percent under - 50, 92.6 percent 50+, 72.2 percent all sizes); establishment-weighted, so a lower estimate. Seeded - blake2b draws keyed by source identity; support clones agree. PolicyEngine-US 2.2.1 subtracts it from - income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, fica_pre_tax_contributions). - Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples) and us_esi_premiums_anchor_gate - (calibrated total within 5 percent of BEA, at most NHE).' + from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI + policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for + active employees, COBRA enrollees and retirees. Column universe: employed (PEMLR 1-2) policyholders + whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated + self-employed and workers without pay carry 0, and their scaled share stays out of the column rather + than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium + and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, + 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier + (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government + and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division + cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 + ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes + the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the + national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): + premium. Some (2): premium minus the contribution of an employee who pays one, which is the published + average contribution (taken over every enrollee, including those who pay nothing) divided by one minus + the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, + United States row by firm size; State cells of those tables are mostly flagged unreliable; Series + III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes + no share of enrollees who pay the whole premium. One factor scales every policyholder''s share so + the weighted anchor-universe total equals CMS NHE Table 24''s employer contribution for the build + year ($1,047.0B CY2024, banked as cms_nhe.cy2024.esi_employer_contribution_premiums), the anchor microcosm#454 + chose; the column carries the employed policyholders'' part. BEA NIPA 7.8 line 17 ($977.0B CY2024 + in the 2026-09-30 vintage, revised from $1,002.9B) is a second estimate of the same active-plus-retired + concept and is recorded beside the verdict, as is MEPS-IC''s own private-sector enrollment total for + active employees. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced + as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The + stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint + from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release + gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the + raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent + of NHE Table 24; employed share within its band).' - stage: prior_year_income survey: CPS ASEC (prior year) source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml index 37502a9688..1ffcd46431 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/spine.yaml @@ -322,10 +322,6 @@ seed_site_bindings: owners: - kind: source_stage id: pregnancy -- site: pre_tax_health_premium_assignment - owners: - - kind: source_stage - id: meps_esi_premiums - site: wic_claim_assignment owners: - kind: source_stage diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 743f2d579d..914c39fe76 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -239,8 +239,8 @@ "evidence": "retired us-data pipeline (archived 2e044571) loss.py line 33; feed bea_nipa families carry no group-health series", "fence": { "origin": "us-data loss.py employer_sponsored_insurance_premiums target", - "purpose": "anchor employer-paid ESI premiums for current employees: the compensation concept the employed-policyholder column carries.", - "verdict_basis": "source-absent from the feed: the meps_esi_premiums stage pins the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., published 2026-09-30: $977.034B CY2024) and scales to it, and the release anchor gate validates against it; compile it here once the ledger banks the series." + "purpose": "anchor employer-paid ESI premiums. The series counts contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19), the same concept as NHE Table 24.", + "verdict_basis": "source-absent from the feed: the meps_esi_premiums stage pins the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., published 2026-09-30: $977.034B CY2024, revised from $1,002.9B) and records the anchor-universe total against it beside the release anchor verdict; the stage scales to NHE Table 24. Compile it here once the ledger banks the series." } }, "bea_nipa.proprietors_income": { @@ -460,23 +460,23 @@ "cms_nhe.esi_employer_contribution_premiums": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "NHE Table 24 employer contribution to ESI premiums. The pe-us input it anchors (employer_sponsored_insurance_premiums) is now produced by the meps_esi_premiums base stage (microcosm#454) for employed current ESI policyholders only, while NHE counts COBRA and retiree enrollees (NHEA methodology paper 2024, p. 28; Table 24 note 1 adds Medicare Retiree Drug Subsidy payments), so a calibration target on this fact would load the retiree share onto workers. Validation-gated instead: us_esi_premiums_anchor_gate holds the calibrated total within 5% of BEA NIPA 7.8 line 17 (the us-data target's concept, pinned in the stage) and at or below this all-employer fact; the private-employer subset rides as the sector cross-check.", - "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: MEPS-IC cells over all 92.4M weighted current policyholders give $1,031.0B (0.985 x NHE $1,047.0B CY2024); the employed universe gives $912.8B raw, scaled to BEA $977.0B (11.5% of the all-policyholder total is non-employed or no-employer coverage); private employers $756.7B vs NHE private $753.4B", + "reason": "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums base stage (microcosm#454) scales to this fact over the fact's own universe, every current ESI policyholder (NHEA methodology paper 2024, p. 29: active employees, COBRA and retirees), and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this all-employer fact; the private-employer subset rides as a sector cross-check.", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $1,047.0B at scale factor 1.040; employed column $925.7B (88.4%); other policyholders $121.3B; private-sector part of the column $713.2B against MEPS-IC's own private active-employee total of $668.6B (2024)", "fence": { "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", - "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NIPA 7.8 line 17 counts employer contributions as compensation of current employees (BEA builds it from MEPS-IC 'insurance purchased by employers for employees'); NHE Table 24 adds COBRA and retiree enrollees, so it bounds the column from above (7.2% above BEA in CY2024).", - "verdict_basis": "deferred, not dropped: the producer is live, but the only banked fact measures a broader concept than the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) uses it as the upper bound today; bank BEA NIPA 7.8 line 17 (B4923C) in the ledger and compile that as the calibration target, and this family stays the cross-check." + "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NHE Table 24 and NIPA 7.8 line 17 both count employer contributions for active and retired employees (NHEA methodology paper; BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in CY2024 and is recorded beside the gate verdict.", + "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. A calibration target needs an exported anchor-universe quantity or a banked active-employee fact." } }, "cms_nhe.esi_private_employer_contribution_premiums": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "NHE Table 24 employer contribution to ESI premiums. The pe-us input it anchors (employer_sponsored_insurance_premiums) is now produced by the meps_esi_premiums base stage (microcosm#454) for employed current ESI policyholders only, while NHE counts COBRA and retiree enrollees (NHEA methodology paper 2024, p. 28; Table 24 note 1 adds Medicare Retiree Drug Subsidy payments), so a calibration target on this fact would load the retiree share onto workers. Validation-gated instead: us_esi_premiums_anchor_gate holds the calibrated total within 5% of BEA NIPA 7.8 line 17 (the us-data target's concept, pinned in the stage) and at or below this all-employer fact; the private-employer subset rides as the sector cross-check.", - "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: MEPS-IC cells over all 92.4M weighted current policyholders give $1,031.0B (0.985 x NHE $1,047.0B CY2024); the employed universe gives $912.8B raw, scaled to BEA $977.0B (11.5% of the all-policyholder total is non-employed or no-employer coverage); private employers $756.7B vs NHE private $753.4B", + "reason": "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums base stage (microcosm#454) scales to this fact over the fact's own universe, every current ESI policyholder (NHEA methodology paper 2024, p. 29: active employees, COBRA and retirees), and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this all-employer fact; the private-employer subset rides as a sector cross-check.", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $1,047.0B at scale factor 1.040; employed column $925.7B (88.4%); other policyholders $121.3B; private-sector part of the column $713.2B against MEPS-IC's own private active-employee total of $668.6B (2024)", "fence": { "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", - "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NIPA 7.8 line 17 counts employer contributions as compensation of current employees (BEA builds it from MEPS-IC 'insurance purchased by employers for employees'); NHE Table 24 adds COBRA and retiree enrollees, so it bounds the column from above (7.2% above BEA in CY2024).", - "verdict_basis": "deferred, not dropped: the producer is live, but the only banked fact measures a broader concept than the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) uses it as the upper bound today; bank BEA NIPA 7.8 line 17 (B4923C) in the ledger and compile that as the calibration target, and this family stays the cross-check." + "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NHE Table 24 and NIPA 7.8 line 17 both count employer contributions for active and retired employees (NHEA methodology paper; BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in CY2024 and is recorded beside the gate verdict.", + "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. A calibration target needs an exported anchor-universe quantity or a banked active-employee fact." } }, "cms_nhe.medicaid_title_xix_expenditures": { diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 10cd7c5adf..c2085f5546 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -267,13 +267,11 @@ ) from microcosm.build.us_runtime.esi_premiums import ( US_ESI_EMPLOYER_PREMIUM_COLUMN, - US_ESI_PRE_TAX_PREMIUM_COLUMN, US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, US_ESI_PREMIUMS_OUTPUT_COLUMNS, US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, US_ESI_PREMIUMS_STAGE_NAME, derive_us_employer_esi_premiums_from_manifest, - derive_us_pre_tax_health_insurance_premiums_from_manifest, load_meps_ic_esi_premium_cells, us_esi_premiums_anchor_gate, us_esi_premiums_signal_gate, @@ -1372,9 +1370,7 @@ "US_ESI_PREMIUMS_OUTPUT_COLUMNS", "US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS", "US_ESI_PREMIUMS_STAGE_NAME", - "US_ESI_PRE_TAX_PREMIUM_COLUMN", "derive_us_employer_esi_premiums_from_manifest", - "derive_us_pre_tax_health_insurance_premiums_from_manifest", "load_meps_ic_esi_premium_cells", "us_esi_premiums_anchor_gate", "us_esi_premiums_signal_gate", @@ -2169,11 +2165,11 @@ def to_manifest(self) -> dict[str, object]: survey="MEPS-IC + CPS ASEC current employment-based coverage", source="https://meps.ahrq.gov/mepsweb/survey_comp/Insurance.jsp", notes=( - "Employed current ESI policyholders take the MEPS-IC employer " - "share of their tier x firm-size x State (or government " - "census-division) cell, scaled to BEA NIPA 7.8 line 17; reported " - "premiums route through payroll at MEPS-IC pretax-offer rates " - "(microcosm #454). Shared by support clones." + "Current ESI policyholders take the MEPS-IC employer share of " + "their tier x firm-size x State (or government census-division) " + "cell, scaled over every policyholder to CMS NHE Table 24 and " + "written for the employed ones (microcosm #454). Deterministic; " + "shared by support clones." ), ), "aca_marketplace_inputs": DonorSpec( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md index 14d5a545a4..f710dac2d3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md @@ -1,7 +1,7 @@ # MEPS-IC employer-premium cells `meps_ic_esi_premium_cells.json` holds the MEPS-IC averages the -`meps_esi_premiums` stage assigns to employed ESI policyholders +`meps_esi_premiums` stage assigns to ESI policyholders (`microcosm.build.us_runtime.esi_premiums`, [PolicyEngine/microcosm#454](https://github.com/PolicyEngine/microcosm/issues/454); method in `docs/us-esi-employer-premiums.md`). @@ -15,7 +15,8 @@ suppressed cell is stage logic, not data. | `private_state_2025` | MEPS-IC 2025 Series II, Tables II.C.1/2, II.D.1/2, II.E.1/2 | Average total premium and average employee contribution per enrolled employee at private-sector establishments, single / family / employee-plus-one, by State, for all firms, under 50 and 50 or more employees | | `private_national_2024` | MEPS-IC 2024 Series II, United States rows | The same six national averages a year earlier; they age the government cells | | `public_division_2024` | MEPS-IC 2024 Series III, Tables III.C.1/2, III.D.1/2, III.E.1/2 | The same averages for State and local government jobs, by census division, for all governments and State governments. 2024 is the latest year AHRQ has published for this series | -| `pretax_contribution_2025` | MEPS-IC 2025 Series I, Tables I.A.2 and I.A.2.j | Percent of private-sector establishments that offer health insurance, and that offer pretax employee contributions to it, by firm size | +| `no_contribution_share_2025` | MEPS-IC 2025 Series II, Tables II.C.4.a, II.D.4.a, II.E.4.a, United States row | Percent of enrollees whose coverage required no employee contribution, by tier, for all firms, under 50 and 50 or more employees. National only: most State cells of these tables are flagged unreliable or suppressed | +| `private_enrollment_national` | MEPS-IC 2024 and 2025 Series II, Tables II.B.1, II.B.2, II.B.2.b, II.C.4, II.D.4, II.E.4, United States row | Private-sector employees, the percent in establishments that offer health insurance, the percent enrolled there, and each tier's share of enrollees: the inputs of the active-employee cross-check | | `state_census_division` | the division headings of Table II.C.1 | State FIPS to census division | `sources` pins each PDF by URL, byte length and SHA-256. @@ -26,15 +27,16 @@ suppressed cell is stage logic, not data. uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic ``` -The tool downloads the four pinned PDFs (about 34 MB) into `--pdf-dir`, +The tool downloads the three pinned PDFs (about 25 MB) into `--pdf-dir`, refuses any whose length or digest differs from its pin, extracts text with `pdftotext -layout` (poppler) and reads one row per State, division or firm-size label. It refuses a label that matches no line or more than one, a row with the wrong number of values, a table whose title names another survey year, and a United States row that differs from the AHRQ national figure it must reproduce (for example single $9,025 and family $26,281 in 2025; the -2024 rows of MEPS-IC Research Findings #54). `--check` regenerates in memory -and fails if the committed file differs. +2024 rows of MEPS-IC Research Findings #54). A national row whose kept column +is suppressed or flagged unreliable is refused too. `--check` regenerates in +memory and fails if the committed file differs. The stage pins this file's own SHA-256 (`esi_premiums._CELLS_SHA256`) and refuses to load any other bytes, so a regeneration must be re-pinned there diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json b/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json index 6e409ee3c2..c13c90de62 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/meps_ic_esi_premium_cells.json @@ -1,29 +1,127 @@ { - "description": "MEPS-IC average total premium and average employee contribution per enrolled employee, by coverage tier, for the meps_esi_premiums source stage: private-sector cells by State and firm size (2025), State and local government cells by census division (2024, the latest published year), the 2024 private national rows that age the government cells, and the 2025 firm-size pretax-contribution offer rates. Values are copied as published; null marks a cell AHRQ suppressed ('--'). Dollar values are annual; percent values are percents.", + "description": "MEPS-IC average total premium and average employee contribution per enrolled employee, by coverage tier, for the meps_esi_premiums source stage: private-sector cells by State and firm size (2025), State and local government cells by census division (2024, the latest published year), the 2024 private national rows that age the government cells, the 2025 national share of enrollees whose coverage required no employee contribution, and the 2024 and 2025 national private-sector enrollment rows. Values are copied as published; null marks a cell AHRQ suppressed ('--'). Dollar values are annual; percent values are percents; employee counts are persons.", "extraction": { "method": "pdftotext -layout, one row per label, refusing any label that does not match exactly one line", "pdftotext": "pdftotext version 26.09.0" }, "generator": "tools/build_us_meps_ic_esi_cells.py", "issue": "PolicyEngine/microcosm#454", - "pretax_contribution_2025": { - "offer_title": "Table I.A.2 Percent of private-sector establishments that offer health insurance by firm size and selected characteristics: United States,", - "rows": { - "50plus": { - "health_insurance_offer_percent": 94.8, - "pretax_contribution_offer_percent": 87.8 + "no_contribution_share_2025": { + "employee_plus_one": { + "50plus": 3.6, + "lt50": 23.1, + "text_line": 18025, + "title": "Table II.E.4.a Percent of private-sector employees enrolled in a health insurance plan that take employee-plus-one coverage that required no employee contribution by firm size and State: United States, 2025", + "total": 6.0 + }, + "family": { + "50plus": 4.6, + "lt50": 28.6, + "text_line": 15679, + "title": "Table II.D.4.a Percent of private-sector employees enrolled in a health insurance plan that take family coverage that required no employee contribution by firm size and State: United States, 2025", + "total": 7.7 + }, + "single": { + "50plus": 8.7, + "lt50": 35.5, + "text_line": 11829, + "title": "Table II.C.4.a Percent of private-sector employees enrolled in a health insurance plan that take single coverage that required no employee contribution by firm size and State: United States, 2025", + "total": 13.0 + } + }, + "private_enrollment_national": { + "2024": { + "employees": { + "50plus": 102965293.0, + "lt50": 36475784.0, + "text_line": 3322, + "title": "Table II.B.1 Number of private-sector employees by firm size and State: United States, 2024", + "total": 139441077.0 + }, + "enrolled_percent": { + "50plus": 55.8, + "lt50": 53.0, + "text_line": 4150, + "title": "Table II.B.2.b Percent of private-sector employees that are enrolled in health insurance at establishments that offer health insurance by firm size and State: United States, 2024", + "total": 55.3 }, - "lt50": { - "health_insurance_offer_percent": 32.1, - "pretax_contribution_offer_percent": 15.9 + "offer_percent": { + "50plus": 97.4, + "lt50": 50.5, + "text_line": 3650, + "title": "Table II.B.2 Percent of private-sector employees in establishments that offer health insurance by firm size and State: United States, 2024", + "total": 85.1 }, - "total": { - "health_insurance_offer_percent": 49.2, - "pretax_contribution_offer_percent": 35.5 + "tier_share_percent": { + "employee_plus_one": { + "50plus": 18.1, + "lt50": 14.3, + "text_line": 17851, + "title": "Table II.E.4 Percent of private-sector employees enrolled in a health insurance plan that take employee-plus-one coverage by firm size and State: United States, 2024", + "total": 17.6 + }, + "family": { + "50plus": 24.4, + "lt50": 19.8, + "text_line": 15507, + "title": "Table II.D.4 Percent of private-sector employees enrolled in a health insurance plan that take family coverage by firm size and State: United States, 2024", + "total": 23.7 + }, + "single": { + "50plus": 57.5, + "lt50": 66.0, + "text_line": 11661, + "title": "Table II.C.4 Percent of private-sector employees enrolled in a health insurance plan that take single coverage by firm size and State: United States, 2024", + "total": 58.7 + } } }, - "text_line": 2376, - "title": "Table I.A.2.j Percent of private-sector establishments that offer tax-advantaged benefits for contributions to health insurance, Flexible" + "2025": { + "employees": { + "50plus": 105297602.0, + "lt50": 35753890.0, + "text_line": 3330, + "title": "Table II.B.1 Number of private-sector employees by firm size and State: United States, 2025", + "total": 141051492.0 + }, + "enrolled_percent": { + "50plus": 54.7, + "lt50": 52.3, + "text_line": 4156, + "title": "Table II.B.2.b Percent of private-sector employees that are enrolled in health insurance at establishments that offer health insurance by firm size and State: United States, 2025", + "total": 54.3 + }, + "offer_percent": { + "50plus": 95.6, + "lt50": 50.4, + "text_line": 3658, + "title": "Table II.B.2 Percent of private-sector employees in establishments that offer health insurance by firm size and State: United States, 2025", + "total": 84.2 + }, + "tier_share_percent": { + "employee_plus_one": { + "50plus": 18.7, + "lt50": 14.9, + "text_line": 17857, + "title": "Table II.E.4 Percent of private-sector employees enrolled in a health insurance plan that take employee-plus-one coverage by firm size and State: United States, 2025", + "total": 18.1 + }, + "family": { + "50plus": 24.2, + "lt50": 21.6, + "text_line": 15511, + "title": "Table II.D.4 Percent of private-sector employees enrolled in a health insurance plan that take family coverage by firm size and State: United States, 2025", + "total": 23.8 + }, + "single": { + "50plus": 57.1, + "lt50": 63.5, + "text_line": 11663, + "title": "Table II.C.4 Percent of private-sector employees enrolled in a health insurance plan that take single coverage by firm size and State: United States, 2025", + "total": 58.1 + } + } + } }, "private_national_2024": { "employee_plus_one": { @@ -2005,18 +2103,10 @@ } } }, - "schema_version": 1, + "schema_version": 2, "sources": { - "private_national_2025": { - "description": "MEPS-IC 2025 Series I (private sector, national): Table I.A.2 (percent of establishments that offer health insurance) and Table I.A.2.j (percent of establishments that offer pretax employee contributions to health insurance), by firm size", - "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", - "sha256": "cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3", - "size_bytes": 9210366, - "survey_year": 2025, - "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/national/series_1/2025/ic25_ia_g.pdf" - }, "private_state_2024": { - "description": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025", + "description": "MEPS-IC 2024 Series II (private sector, by firm size and State); the United States rows age the 2024 State and local government cells to 2025 and carry the 2024 private-sector enrollment", "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", "sha256": "4ae0dada70fac7c147508b946a2d00cf2e8930f5550f6beb9d8a1bf3de71d20b", "size_bytes": 10687163, @@ -2024,7 +2114,7 @@ "url": "https://meps.ahrq.gov/mepsweb/data_stats/summ_tables/insr/state/series_2/2024/ic24_iia_f.pdf" }, "private_state_2025": { - "description": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee) and II.C/D/E.2 (average employee contribution)", + "description": "MEPS-IC 2025 Series II (private sector, by firm size and State): Tables II.C/D/E.1 (average total premium per enrolled employee), II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent of enrollees whose coverage required no employee contribution), and the enrollment rows II.B.1, II.B.2, II.B.2.b and II.C/D/E.4", "publisher": "Agency for Healthcare Research and Quality, Center for Financing, Access and Cost Trends, Medical Expenditure Panel Survey-Insurance Component", "sha256": "ebebf083d132e2b5a72ac99d659ee6d46a382280a47d2ff2d20f6e9e25b69330", "size_bytes": 12530005, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py index 4c1960ff94..dc2c722a97 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py @@ -1,16 +1,9 @@ -"""MEPS-IC employer premiums and pre-tax employee premiums for ESI policyholders. +"""MEPS-IC employer premiums for ESI policyholders. -Microcosm #454. PolicyEngine-US 2.2.1 reads two person inputs that no -Microcosm stage produced: - -* ``employer_sponsored_insurance_premiums`` — "annual employer-paid health - insurance premiums", one of ``gov.household.cbo_market_income_additions`` - (CBO household market income); and -* ``pre_tax_health_insurance_premiums`` — premiums paid through pre-tax - payroll deductions, listed in ``gov.irs.gross_income.pre_tax_contributions`` - (subtracted from ``employment_income`` in ``irs_employment_income``) and - ``gov.irs.gross_income.fica_pre_tax_contributions`` (subtracted in - ``payroll_tax_gross_wages``). +Microcosm #454. PolicyEngine-US 2.2.1 reads a person input that no Microcosm +stage produced: ``employer_sponsored_insurance_premiums``, "annual +employer-paid health insurance premiums", one of +``gov.household.cbo_market_income_additions`` (CBO household market income). The retired eCPS derivation (archived 42ed5d45 ``datasets/cps/cps.py`` L197-271) read the CPS ASEC current employment-based coverage fields, which the @@ -19,12 +12,18 @@ Employer premium ---------------- -The universe is the **employed** (``PEMLR`` 1-2) **current ESI policyholder** -(``NOW_OWNGRP`` 1). Dependents carry nothing (the policyholder carries the -premium) and non-employed policyholders carry nothing (retiree and COBRA -coverage is not compensation of a current job). Self-employed-unincorporated -workers and workers without pay (``PEIO1COW`` 7-8) carry nothing either: their -current job has no employer to pay a share. +Both national accounts that publish an employer-contribution total (CMS NHE +Table 24 and BEA NIPA Table 7.8 line 17) count contributions for active +**and** retired employees, and both build the non-federal part from MEPS-IC. +So the stage prices every **current ESI policyholder** (``NOW_OWNGRP`` 1, +employed or not: the accounts' universe), scales that universe to the account +total, and writes the result only for the **employed** policyholders +(``PEMLR`` 1-2) whose current job has an employer (``PEIO1COW`` 1-6). The +column is the compensation of a current job: dependents carry nothing (the +policyholder carries the premium), and retired, unemployed and other +non-employed policyholders, self-employed-unincorporated workers and workers +without pay carry nothing either. Their scaled share stays out of the column +rather than being loaded onto workers. Each policyholder's raw employer share comes from the MEPS-IC cell of their coverage tier (``NOW_GRPFTYP2``: family, self plus one, self-only), employer @@ -33,9 +32,15 @@ * employer paid all of the premium (``NOW_HIPAID`` 1): the cell's average total premium; -* employer paid some (2): the average total premium minus the average employee - contribution; -* employer paid none (3): zero. +* employer paid some (2): the average total premium minus the contribution of + an employee who pays one. MEPS-IC averages the employee contribution over + every enrollee, including those who pay nothing, so the stage divides it by + one minus the published share of enrollees whose coverage required no + contribution (Tables II.C/D/E.4.a, national by firm size; most State cells + of those tables are flagged unreliable). A cohort with MEPS-IC's own + all/some mix then reproduces the cell's published employer mean; +* employer paid none (3): zero. MEPS-IC publishes no share of enrollees who + pay the whole premium, so they are treated as outside its averages. Private-sector cells are MEPS-IC 2025 Series II (State by firm size). Government cells are MEPS-IC 2024 Series III by census division (the latest @@ -43,33 +48,36 @@ the same tier and measure: State government employees take the State-government column, local and federal employees the all-governments column (MEPS-IC does not survey the federal government, so that is a -stand-in). AHRQ suppresses nine small-firm employee-contribution cells; each -takes the national small-firm cell times the State's total-to-national ratio. - -A single factor then scales every raw share so the weighted total equals BEA -NIPA Table 7.8 line 17 (series B4923C, "Private group health insurance", -employer contributions as a supplement to wages and salaries; government -employers' contributions to privately administered plans included) for the -build year. BEA's NIPA handbook (chapter 10, table 10.B) says the private and -State and local parts of that series come from MEPS data "on insurance -purchased by employers for employees", the active-employee concept this column -carries. CMS NHE Table 24's employer contribution is the cross-check, and an -upper bound: the NHEA methodology counts premiums for active employees, COBRA -enrollees and retirees. - -Pre-tax employee premium ------------------------- -Eligible: an employed policyholder with wages last year (``WSAL_VAL`` > 0), a -positive reported premium (``PHIP_VAL``) and an employee share to pay -(``NOW_HIPAID`` 2 or 3). An eligible person pays the reported premium pre-tax -with the firm-size probability that their employer offers pretax employee -contributions, among employers that offer health insurance: MEPS-IC 2025 Table -I.A.2.j over Table I.A.2. The rates are establishment-weighted; the published -bands rise with firm size (8.9% under 10 employees, 92.6% at 1,000 or more), -so they understate the enrollee-weighted rate and the column is a lower -estimate. -The draw is a seeded blake2b uniform keyed by stable source identity, so -support clones agree. Others carry zero. +stand-in). Series III publishes no no-contribution share, so government cells +take the private 50-or-more share (a stand-in). A policyholder with no +employer class (not employed and never asked, self-employed unincorporated or +without pay) takes the State's all-sizes private cell. No MEPS-IC table prices +retiree coverage, so non-employed policyholders are priced as active +employees; if retiree plans cost less, the employed column is understated. +AHRQ suppresses nine small-firm employee-contribution cells; each takes the +national small-firm cell times the State's total-to-national ratio. + +One factor scales every policyholder's raw share so the weighted total equals +CMS NHE Table 24's employer contribution to private health insurance premiums +for the build year, the anchor microcosm#454 chose. Its methodology paper says +the estimate covers "active employees, continuation of health coverage +(COBRA), and retirees" of private and State and local sponsors (from MEPS-IC) +and federal "employers, employees and retirees" (from OPM). BEA NIPA Table 7.8 +line 17 (series B4923C) is a second estimate of the same concept, 6.7% lower +for 2024: BEA's State Personal Income methodology (December 2025, paragraphs +3.18-3.19) says its MEPS source "covers both health insurance purchased by +employers for their active and retired employees" and its federal estimate +covers "active and retired federal civilian employees". It is recorded as a +cross-check, not gated. + +Not produced here +----------------- +``pre_tax_health_insurance_premiums``, the employee share paid by pre-tax +payroll deduction, has no producer yet. PolicyEngine-US treats it as disjoint +from the other premium inputs (PolicyEngine/policyengine-us#10046), so +producing it means moving those dollars out of the reported premium the base +carries, on an engine that counts the pre-tax input where the reported premium +was counted. That is a separate change. """ from __future__ import annotations @@ -106,10 +114,10 @@ "US_ESI_PREMIUMS_OUTPUT_COLUMNS", "US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS", "US_ESI_PREMIUMS_STAGE_NAME", - "US_ESI_PRE_TAX_PREMIUM_COLUMN", "derive_us_employer_esi_premiums_from_manifest", - "derive_us_pre_tax_health_insurance_premiums_from_manifest", "load_meps_ic_esi_premium_cells", + "meps_ic_private_active_employer_totals", + "refuse_unassigned_us_esi_premiums", "us_esi_premiums_anchor_gate", "us_esi_premiums_signal_gate", "us_esi_premiums_stage_spec", @@ -119,11 +127,7 @@ US_ESI_PREMIUMS_STAGE_NAME = "meps_esi_premiums" US_ESI_EMPLOYER_PREMIUM_COLUMN = "employer_sponsored_insurance_premiums" -US_ESI_PRE_TAX_PREMIUM_COLUMN = "pre_tax_health_insurance_premiums" -US_ESI_PREMIUMS_OUTPUT_COLUMNS: tuple[str, ...] = ( - US_ESI_EMPLOYER_PREMIUM_COLUMN, - US_ESI_PRE_TAX_PREMIUM_COLUMN, -) +US_ESI_PREMIUMS_OUTPUT_COLUMNS: tuple[str, ...] = (US_ESI_EMPLOYER_PREMIUM_COLUMN,) US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS = US_ESI_PREMIUMS_OUTPUT_COLUMNS #: Raw CPS ASEC person columns the stage reads (the six NOW_*/PEMLR/NOEMP @@ -138,8 +142,6 @@ "PEMLR", "NOEMP", "PEIO1COW", - "PHIP_VAL", - "WSAL_VAL", "state_fips", ) @@ -182,24 +184,44 @@ #: nonprofit, 6 self-employed incorporated, 7 self-employed unincorporated, #: 8 without pay. _FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT = 1, 2, 3 +_GOVERNMENT_EMPLOYER_CODES = (_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT) _PRIVATE_EMPLOYER_CODES = (4, 5, 6) _EMPLOYER_SECTOR_CODES = (_FEDERAL, _STATE_GOVERNMENT, _LOCAL_GOVERNMENT, 4, 5, 6) _CELLS_RESOURCE = "meps_ic_esi_premium_cells.json" -_CELLS_SHA256 = "fb07b8c27ff94fd1423623f7385868fb26e0fce7ad99f80c7cf0d205364e77f8" +_CELLS_SHA256 = "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93" _PERSON_WEIGHT_COLUMN = "person_weight" -_DRAW_SALT = US_ESI_PRE_TAX_PREMIUM_COLUMN _BEA_SECTION7_URL = ( "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx" ) _NHE_TABLES_URL = "https://www.cms.gov/files/zip/nhe-tables.zip" +#: CMS NHE Table 24 (2024 National Health Expenditure Accounts, data through +#: CY2024), row "Employer Contribution to Private Health Insurance Premiums": +#: private, federal and State and local employers, for active employees, COBRA +#: enrollees and retirees. The anchor microcosm#454 chose; banked in the +#: pinned feed as cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums. +EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( + { + "source": _NHE_TABLES_URL, + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "row": "Employer Contribution to Private Health Insurance Premiums", + "coverage": ( + "active employees, COBRA enrollees and retirees of private, " + "federal and State and local employers" + ), + # Integer dollars: the manifest round-trips through YAML. + "values": {"2023": 975_700_000_000, "2024": 1_047_000_000_000}, + } +) #: BEA NIPA Table 7.8 line 17 (B4923C), millions of dollars converted to #: dollars, from the annual update published 2026-09-30 (file created #: 2026-09-28). That update revised 2024 from the $1,002.9B the retired -#: us-data loss matrix pinned. -EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( +#: us-data loss matrix pinned. A second estimate of the same concept; recorded, +#: not gated. +EMPLOYER_PREMIUM_CROSS_CHECK: Mapping[str, Any] = MappingProxyType( { "source": _BEA_SECTION7_URL, "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", @@ -208,53 +230,43 @@ "series": "B4923C", "label": "Private group health insurance", "published": "2026-09-30", - # Integer dollars: the manifest round-trips through YAML. "values": { "2023": 923_195_000_000, "2024": 977_034_000_000, "2025": 1_027_929_000_000, }, - } -) -#: CMS NHE Table 24 (2024 release, data through CY2024), the cross-check and -#: upper bound; banked in the pinned feed as -#: cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums and -#: ..._private_employer_contribution_premiums. -EMPLOYER_PREMIUM_CROSS_CHECK: Mapping[str, Any] = MappingProxyType( - { - "source": _NHE_TABLES_URL, - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "table": "Table 24 Employer-Sponsored Private Health Insurance", - "employer_contribution": {"2023": 975.7e9, "2024": 1_047.0e9}, - "employee_contribution": {"2023": 357.8e9, "2024": 382.1e9}, "relation": ( - "upper bound: NHEA counts employer contributions for active " - "employees, COBRA enrollees and retirees" + "second estimate of the same concept: employer contributions for " + "active and retired employees (BEA State Personal Income " + "methodology, December 2025, paragraphs 3.18-3.19)" ), } ) -#: Tolerance of the calibrated release aggregate around the BEA anchor. The -#: stage pins the pre-calibration aggregate exactly; the band absorbs -#: calibration and sparse-selection drift. It is wider than BEA's own 2024 -#: revision (2.6%) and narrower than the NHE-BEA gap (7.2%), so a release that -#: drifts toward the retiree-inclusive NHE concept fails. +#: Tolerance of the calibrated release aggregate around the anchor, over the +#: anchor's universe (every policyholder). The stage pins the pre-calibration +#: aggregate exactly, so the band is the room a release gives calibration and +#: sparse selection to move it. It is a chosen release criterion, not a +#: measured error: no release has calibrated this column yet. ANCHOR_RELATIVE_TOLERANCE = 0.05 -#: Weighted-raw employer share per positive holder, before scaling. MEPS-IC -#: 2025 employer shares run $7.2k (single) to $19.0k (family) and the pinned -#: pools measure $12.3-12.4k; this band catches a broken cell table, which the -#: scale factor would otherwise hide. +#: Weighted-raw employer share per positive employed holder, before scaling. +#: MEPS-IC 2025 employer shares run $7.2k (single) to $19.0k (family) and the +#: pinned pools measure $12.0-12.1k. The band catches a gross unit error in the +#: cell table (monthly for annual, cents for dollars), which the scale factor +#: would otherwise hide; it does not catch a level error inside the band. _RAW_MEAN_BAND = (6_000.0, 20_000.0) #: Weighted share of all persons with a positive employer premium. Measured #: on both pinned pools: 22.7%. _POSITIVE_SHARE_BAND = (0.15, 0.32) -#: Weighted share of all persons with a positive pre-tax premium. Measured on -#: both pinned pools: 16.1%. -_PRE_TAX_POSITIVE_SHARE_BAND = (0.05, 0.25) +#: Employed policyholders' share of the anchor-universe employer total. +#: Measured on the pinned pools: 88.4% and 88.2%. A release whose calibration +#: moves the split far from that has reweighted workers against retirees. +_EMPLOYED_SHARE_BAND = (0.80, 0.95) _EMPLOYER_PREMIUM_PARAMETERS: Mapping[str, Any] = MappingProxyType( { "cells_resource": f"microcosm.build.us_runtime.data/{_CELLS_RESOURCE}", "cells_sha256": _CELLS_SHA256, + "anchor_universe": "NOW_OWNGRP == 1", "universe": ( "NOW_OWNGRP == 1 and PEMLR in (1, 2) and PEIO1COW in (1, 2, 3, 4, 5, 6)" ), @@ -264,7 +276,15 @@ "payment_column": "NOW_HIPAID", "payment_rule": ( "1 (employer paid all): average total premium; 2 (some): average " - "total premium minus average employee contribution; 3 (none): 0" + "total premium minus average employee contribution / (1 - " + "no-contribution share); 3 (none): 0" + ), + "no_contribution_share": ( + "MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; " + "government cells take the 50-or-more share" + ), + "other_policyholder_cell": ( + "PEIO1COW outside 1-6: the State's all-sizes private cell" ), "suppressed_cell_fallback": ( "national firm-size cell times the State total-to-national ratio" @@ -273,26 +293,19 @@ "MEPS-IC 2024 Series III x private-sector national 2025/2024 " "ratio of the same tier and measure" ), + "scaling": ( + "one factor, anchor / weighted raw share of the anchor universe; " + "written to the universe only" + ), "anchor": { "source": EMPLOYER_PREMIUM_ANCHOR["source"], "sha256": EMPLOYER_PREMIUM_ANCHOR["sha256"], - "series": EMPLOYER_PREMIUM_ANCHOR["series"], + "table": EMPLOYER_PREMIUM_ANCHOR["table"], + "row": EMPLOYER_PREMIUM_ANCHOR["row"], "values": dict(EMPLOYER_PREMIUM_ANCHOR["values"]), }, } ) -_PRE_TAX_PARAMETERS: Mapping[str, Any] = MappingProxyType( - { - "seed_from_build_config": True, - "premium_column": "PHIP_VAL", - "wage_column": "WSAL_VAL", - "employer_payment_codes": [_PAYS_SOME, _PAYS_NONE], - "pre_tax_share": ( - "MEPS-IC 2025 Table I.A.2.j pretax-contribution offer percent / " - "Table I.A.2 health-insurance offer percent, by firm size" - ), - } -) def us_esi_premiums_stage_spec() -> SourceStageSpec: @@ -316,7 +329,6 @@ def us_esi_premiums_stage_spec() -> SourceStageSpec: "derive_employer_sponsored_insurance_premiums", _plain(_EMPLOYER_PREMIUM_PARAMETERS), ), - ("derive_pre_tax_health_insurance_premiums", _plain(_PRE_TAX_PARAMETERS)), ) actual = tuple( (operation.kind, dict(operation.parameters)) for operation in spec.operations @@ -324,7 +336,7 @@ def us_esi_premiums_stage_spec() -> SourceStageSpec: if actual != expected: raise ValueError( "US ESI premium operations drifted from the reviewed MEPS-IC cell " - "assignment and pre-tax share contract." + "assignment contract." ) return spec @@ -362,10 +374,18 @@ class _EsiCodes: sector: np.ndarray state_fips: np.ndarray + @property + def policyholder(self) -> np.ndarray: + """Every current ESI policyholder: the universe the anchor measures.""" + + return self.owner == _POLICYHOLDER + @property def universe(self) -> np.ndarray: + """Employed policyholders with an employer: the column's universe.""" + return ( - (self.owner == _POLICYHOLDER) + self.policyholder & self.employed & np.isin(self.sector, _EMPLOYER_SECTOR_CODES) ) @@ -488,16 +508,21 @@ def _government_cell( return float(value) * float(current) / float(prior) +def _no_contribution_share(cells: Mapping[str, Any], tier: str, size: str) -> float: + return float(cells["no_contribution_share_2025"][tier][size]) / 100.0 + + def _cell_values( codes: _EsiCodes, cells: Mapping[str, Any] -) -> tuple[np.ndarray, np.ndarray]: - """Average total premium and employee contribution of each person's cell. +) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Each policyholder's cell: premium, employee contribution, no-contribution share. - NaN outside the employed-policyholder universe. + NaN for everyone who is not a policyholder. """ premium = np.full(len(codes.owner), np.nan) contribution = np.full(len(codes.owner), np.nan) + no_contribution = np.full(len(codes.owner), np.nan) divisions = cells["state_census_division"] keys = pd.DataFrame( { @@ -507,9 +532,10 @@ def _cell_values( "state": codes.state_fips, } ) - universe = codes.universe for (tier_code, noemp, sector, state), index in ( - keys[universe].groupby(["tier", "noemp", "sector", "state"]).groups.items() + keys[codes.policyholder] + .groupby(["tier", "noemp", "sector", "state"]) + .groups.items() ): tier = _TIER_BY_CODE[int(tier_code)] state_key = f"{int(state):02d}" @@ -517,43 +543,120 @@ def _cell_values( raise SourceRuntimeError( f"US ESI premium stage: State FIPS {int(state)} has no MEPS-IC cell." ) - if int(sector) in _PRIVATE_EMPLOYER_CODES: - size = _SIZE_BY_NOEMP[int(noemp)] + if int(sector) in _GOVERNMENT_EMPLOYER_CODES: + column = ( + "state" if int(sector) == _STATE_GOVERNMENT else "all_state_and_local" + ) values = [ - _private_cell(cells, tier, measure, state_key, size) + _government_cell(cells, tier, measure, divisions[state_key], column) for measure in ("premium", "employee_contribution") ] + # Series III publishes no no-contribution share. + size = "50plus" else: - column = ( - "state" if int(sector) == _STATE_GOVERNMENT else "all_state_and_local" + # Private employers by firm size; a policyholder with no employer + # class takes the State's all-sizes private cell. + size = ( + _SIZE_BY_NOEMP[int(noemp)] + if int(sector) in _PRIVATE_EMPLOYER_CODES + else "total" ) values = [ - _government_cell(cells, tier, measure, divisions[state_key], column) + _private_cell(cells, tier, measure, state_key, size) for measure in ("premium", "employee_contribution") ] rows = np.asarray(index, dtype=np.int64) premium[rows], contribution[rows] = values - return premium, contribution + no_contribution[rows] = _no_contribution_share(cells, tier, size) + return premium, contribution, no_contribution def _raw_employer_share( - codes: _EsiCodes, premium: np.ndarray, contribution: np.ndarray + codes: _EsiCodes, + premium: np.ndarray, + contribution: np.ndarray, + no_contribution: np.ndarray, ) -> np.ndarray: + """Unscaled employer share of every policyholder (zero for everyone else).""" + raw = np.zeros(len(codes.owner), dtype=np.float64) - universe = codes.universe - pays_all = universe & (codes.hipaid == _PAYS_ALL) - pays_some = universe & (codes.hipaid == _PAYS_SOME) + holder = codes.policyholder + pays_all = holder & (codes.hipaid == _PAYS_ALL) + pays_some = holder & (codes.hipaid == _PAYS_SOME) raw[pays_all] = premium[pays_all] - raw[pays_some] = np.clip(premium[pays_some] - contribution[pays_some], 0.0, None) + # The published contribution averages over enrollees who pay nothing; + # dividing by the share who pay gives the contribution of those who do. + raw[pays_some] = np.clip( + premium[pays_some] + - contribution[pays_some] / (1.0 - no_contribution[pays_some]), + 0.0, + None, + ) return raw +def _person_raw_shares( + person: pd.DataFrame, +) -> tuple[_EsiCodes, np.ndarray, np.ndarray, np.ndarray]: + """Codes, raw employer share, cell premium and cell employee contribution.""" + + codes = _esi_person_codes(person) + premium, contribution, no_contribution = _cell_values( + codes, load_meps_ic_esi_premium_cells() + ) + raw = _raw_employer_share(codes, premium, contribution, no_contribution) + return codes, raw, premium, contribution + + +def meps_ic_private_active_employer_totals( + cells: Mapping[str, Any], +) -> dict[str, float]: + """MEPS-IC's own private-sector employer total for enrolled employees. + + Enrolled employees (employees x the share in establishments that offer + health insurance x the share enrolled there) times each tier's share of + enrollees and its average premium less average employee contribution, + from the United States rows of the pinned Series II tables. It covers + active private-sector employees only, so it checks the private part of + the column against the survey the cells come from. + """ + + national = { + "2024": lambda tier, measure: cells["private_national_2024"][tier][measure][ + "total" + ], + "2025": lambda tier, measure: cells["private_state_2025"][tier][measure][ + "rows" + ]["US"]["total"], + } + totals: dict[str, float] = {} + for year, rows in cells["private_enrollment_national"].items(): + enrolled = ( + float(rows["employees"]["total"]) + * float(rows["offer_percent"]["total"]) + / 100.0 + * float(rows["enrolled_percent"]["total"]) + / 100.0 + ) + totals[year] = sum( + enrolled + * float(rows["tier_share_percent"][tier]["total"]) + / 100.0 + * ( + float(national[year](tier, "premium")) + - float(national[year](tier, "employee_contribution")) + ) + for tier in _TIER_BY_CODE.values() + ) + return totals + + def _anchor(year: int) -> float: values = EMPLOYER_PREMIUM_ANCHOR["values"] if str(year) not in values: raise SourceRuntimeError( - f"US ESI premium stage has no BEA anchor for {year}; pinned years: " - f"{sorted(values)}." + f"US ESI premium stage has no NHE Table 24 anchor for {year}; pinned " + f"years: {sorted(values)}." ) return float(values[str(year)]) @@ -584,7 +687,11 @@ def derive_us_employer_esi_premiums_from_manifest( operation: SourceOperationSpec, context: SourceRuntimeContext, ) -> pd.DataFrame: - """Assign MEPS-IC employer shares and scale them to the BEA anchor.""" + """Assign MEPS-IC employer shares and scale them to the NHE anchor. + + The factor is set over every policyholder, the anchor's universe; the + column carries the employed policyholders' part. + """ if operation.kind != "derive_employer_sponsored_insurance_premiums": raise SourceRuntimeError( @@ -593,31 +700,20 @@ def derive_us_employer_esi_premiums_from_manifest( if frame is None: raise SourceRuntimeError("ESI premium assignment requires the person table.") _check_parameters(operation, _EMPLOYER_PREMIUM_PARAMETERS) - codes = _esi_person_codes(frame) - premium, contribution = _cell_values(codes, load_meps_ic_esi_premium_cells()) - raw = _raw_employer_share(codes, premium, contribution) + codes, raw, _premium, _contribution = _person_raw_shares(frame) weights = _person_weights(frame) - raw_total = float(weights @ raw) - if not raw_total > 0: + universe = codes.universe + if not float(weights[universe] @ raw[universe]) > 0: raise SourceRuntimeError( "US ESI premium stage found no weighted employer-paid premium mass to " "scale: no employed policyholder with an employer share." ) - scale = _anchor(context.config.target_year) / raw_total + scale = _anchor(context.config.target_year) / float(weights @ raw) result = frame.copy(deep=True) - result[US_ESI_EMPLOYER_PREMIUM_COLUMN] = raw * scale + result[US_ESI_EMPLOYER_PREMIUM_COLUMN] = np.where(universe, raw * scale, 0.0) return result -def _pre_tax_shares(cells: Mapping[str, Any]) -> dict[str, float]: - rows = cells["pretax_contribution_2025"]["rows"] - return { - size: rows[size]["pretax_contribution_offer_percent"] - / rows[size]["health_insurance_offer_percent"] - for size in ("total", "lt50", "50plus") - } - - def _stable_person_keys(frame: pd.DataFrame) -> pd.Series: if {"source_year", "source_household_id", "source_person_id"} <= set(frame.columns): return ( @@ -630,84 +726,6 @@ def _stable_person_keys(frame: pd.DataFrame) -> pd.Series: return frame["person_id"].astype(str) -def _stable_person_draws(frame: pd.DataFrame, *, seed: int) -> np.ndarray: - """Seeded uniform draws keyed by stable source identity per person. - - Support-channel clones share their source identity, so they always - receive the same draw; frames without source columns key on the person - id itself. - """ - - denominator = float(2**64) - return np.asarray( - [ - int.from_bytes( - hashlib.blake2b( - f"{seed}:{_DRAW_SALT}:{key}".encode(), - digest_size=8, - ).digest(), - byteorder="big", - signed=False, - ) - / denominator - for key in _stable_person_keys(frame) - ], - dtype=np.float64, - ) - - -def _pre_tax_eligible(frame: pd.DataFrame, codes: _EsiCodes) -> np.ndarray: - premium = pd.to_numeric(frame["PHIP_VAL"], errors="coerce").to_numpy( - dtype=np.float64 - ) - wages = pd.to_numeric(frame["WSAL_VAL"], errors="coerce").to_numpy(dtype=np.float64) - if not (np.isfinite(premium).all() and np.isfinite(wages).all()): - raise SourceRuntimeError( - "US ESI premium stage requires finite PHIP_VAL and WSAL_VAL." - ) - return ( - codes.universe - & np.isin(codes.hipaid, [_PAYS_SOME, _PAYS_NONE]) - & (wages > 0) - & (premium > 0) - ) - - -def derive_us_pre_tax_health_insurance_premiums_from_manifest( - frame: pd.DataFrame | None, - operation: SourceOperationSpec, - context: SourceRuntimeContext, -) -> pd.DataFrame: - """Route an eligible policyholder's reported premium through payroll.""" - - if operation.kind != "derive_pre_tax_health_insurance_premiums": - raise SourceRuntimeError( - f"Unexpected ESI premium operation {operation.kind!r}." - ) - if frame is None: - raise SourceRuntimeError( - "Pre-tax premium assignment requires the person table." - ) - _check_parameters(operation, _PRE_TAX_PARAMETERS) - codes = _esi_person_codes(frame) - eligible = _pre_tax_eligible(frame, codes) - shares = _pre_tax_shares(load_meps_ic_esi_premium_cells()) - probability = ( - pd.Series(codes.noemp).map(dict(_SIZE_BY_NOEMP)).map(shares).to_numpy() - ) - draws = _stable_person_draws(frame, seed=context.config.seed) - premium = pd.to_numeric(frame["PHIP_VAL"]).to_numpy(dtype=np.float64) - result = frame.copy(deep=True) - result[US_ESI_PRE_TAX_PREMIUM_COLUMN] = np.where( - eligible & (draws < probability), premium, 0.0 - ) - return result - - -def _has_raw_columns(person: pd.DataFrame) -> bool: - return set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) <= set(person.columns) - - def _person_with_state(frame: Frame) -> pd.DataFrame: """The person table with the household ``state_fips`` joined on.""" @@ -740,9 +758,11 @@ def _outputs_carry_signal(person: pd.DataFrame) -> bool: def with_us_esi_premium_inputs(frame: Frame, *, seed: int, time_period: int) -> Frame: """Run the ``meps_esi_premiums`` stage over a US frame. - A frame that already carries both outputs with signal and passes the - signal gate is returned unchanged (idempotent). Otherwise the person table - must still carry the raw ASEC columns; the stage never defaults them. + A frame that already carries the output with signal and passes the signal + gate is returned unchanged (idempotent). Otherwise the person table must + still carry the raw ASEC columns; the stage never defaults them. The + assignment is deterministic: ``seed`` only fills the runtime config that + every source stage takes. """ if frame.schema != US_SCHEMA: @@ -765,9 +785,6 @@ def with_us_esi_premium_inputs(frame: Frame, *, seed: int, time_period: int) -> "derive_employer_sponsored_insurance_premiums": ( derive_us_employer_esi_premiums_from_manifest ), - "derive_pre_tax_health_insurance_premiums": ( - derive_us_pre_tax_health_insurance_premiums_from_manifest - ), }, config=SourceRuntimeConfig(seed=int(seed), target_year=int(time_period)), ) @@ -788,6 +805,31 @@ def with_us_esi_premium_inputs(frame: Frame, *, seed: int, time_period: int) -> ) +def refuse_unassigned_us_esi_premiums(frame: Frame, *, consumer: str) -> None: + """Refuse a frame whose ESI employer premium input is null on some rows. + + A lineage that pools rows the stage never ran on (ACS-spine rows beside an + ASEC donor that carries the column) leaves it null there. Filling those + with the engine default would ship a column that is zero on part of the + population yet looks populated, and no anchor gate runs on that lineage, so + the consumer refuses instead. + """ + + person = frame.table("person") + gaps = { + column: int(person[column].isna().sum()) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS + if column in person.columns and person[column].isna().any() + } + if gaps: + raise SourceRuntimeError( + f"{consumer}: ESI premium input(s) are null on some rows {gaps}. The " + "meps_esi_premiums stage (microcosm#454) ran on the ASEC rows only; " + "its output is never filled with an engine default. Assign or " + "transfer it for every spine before building this lineage." + ) + + def _weighted( weights: np.ndarray, values: np.ndarray, mask: np.ndarray | None = None ) -> float: @@ -816,18 +858,20 @@ def _clone_disagreements(person: pd.DataFrame) -> int: def us_esi_premiums_summary(frame: Frame) -> dict[str, object]: """Weighted ESI premium lineage and structure, for gates and manifests. - With the raw ASEC columns present it also recomputes the raw MEPS-IC - shares, so the scale factor and every structural zero are checked against - the cells rather than trusted. + It recomputes the raw MEPS-IC shares from the raw ASEC columns, so the + scale factor, the anchor-universe total and every structural zero are + checked against the cells rather than trusted. A frame that lacks those + columns gets the column totals only, with ``source_columns_missing`` + naming what is absent; both gates fail on it. """ person = frame.table("person") weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) - pre_tax = person[US_ESI_PRE_TAX_PREMIUM_COLUMN].to_numpy(dtype=np.float64) positive = employer > 0 total_weight = float(weights.sum()) holders = float(weights[positive].sum()) + person = _person_with_state(frame) summary: dict[str, object] = { "employer_premium_total": _weighted(weights, employer), "employer_premium_positive_persons": holders, @@ -837,35 +881,34 @@ def us_esi_premiums_summary(frame: Frame) -> dict[str, object]: "employer_premium_positive_share": holders / total_weight if total_weight else 0.0, - "pre_tax_premium_total": _weighted(weights, pre_tax), - "pre_tax_premium_positive_persons": float(weights[pre_tax > 0].sum()), - "pre_tax_premium_positive_share": ( - float(weights[pre_tax > 0].sum()) / total_weight if total_weight else 0.0 - ), - "nonfinite_rows": int( - (~np.isfinite(employer)).sum() + (~np.isfinite(pre_tax)).sum() - ), - "negative_rows": int((employer < 0).sum() + (pre_tax < 0).sum()), + "nonfinite_rows": int((~np.isfinite(employer)).sum()), + "negative_rows": int((employer < 0).sum()), "clone_disagreement_source_persons": _clone_disagreements(person), "anchor": dict(EMPLOYER_PREMIUM_ANCHOR), "cross_check": dict(EMPLOYER_PREMIUM_CROSS_CHECK), "cells_sha256": _CELLS_SHA256, + "source_columns_missing": [ + column + for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS + if column not in person.columns + ], } - person = _person_with_state(frame) - if not _has_raw_columns(person): + if summary["source_columns_missing"]: return summary - codes = _esi_person_codes(person) - cells = load_meps_ic_esi_premium_cells() - premium, contribution = _cell_values(codes, cells) - raw = _raw_employer_share(codes, premium, contribution) - raw_positive = raw > 0 - raw_total = _weighted(weights, raw) - scale = summary["employer_premium_total"] / raw_total if raw_total else float("nan") - eligible = _pre_tax_eligible(person, codes) - phip = pd.to_numeric(person["PHIP_VAL"]).to_numpy(dtype=np.float64) - holder = codes.owner == _POLICYHOLDER + codes, raw, premium, contribution = _person_raw_shares(person) + universe = codes.universe + holder = codes.policyholder + other = holder & ~universe + raw_positive = universe & (raw > 0) + raw_universe_total = _weighted(weights, raw, universe) + raw_anchor_total = _weighted(weights, raw) + scale = ( + summary["employer_premium_total"] / raw_universe_total + if raw_universe_total + else float("nan") + ) by_tier = { - tier: _weighted(weights, employer, codes.universe & (codes.tier == code)) + tier: _weighted(weights, employer, universe & (codes.tier == code)) for code, tier in _TIER_BY_CODE.items() } by_sector = { @@ -883,62 +926,79 @@ def us_esi_premiums_summary(frame: Frame) -> dict[str, object]: ratio = np.where( raw_positive, employer / np.where(raw_positive, raw, 1.0), np.nan ) + # MEPS-IC's published employer mean of a cell is premium less average + # contribution over every enrollee the employer pays for. + employer_pays = universe & np.isin(codes.hipaid, [_PAYS_ALL, _PAYS_SOME]) + published = _weighted(weights, premium - contribution, employer_pays) summary |= { "weighted_policyholders": float(weights[holder].sum()), - "weighted_employed_policyholders": float(weights[codes.universe].sum()), + "weighted_employed_policyholders": float(weights[universe].sum()), "weighted_employed_policyholders_by_payment": { - name: float(weights[codes.universe & (codes.hipaid == code)].sum()) + name: float(weights[universe & (codes.hipaid == code)].sum()) for name, code in ( ("all", _PAYS_ALL), ("some", _PAYS_SOME), ("none", _PAYS_NONE), ) }, - "raw_employer_share_total": raw_total, + "weighted_other_policyholders": float(weights[other].sum()), + "weighted_other_policyholders_with_employer_share": float( + weights[other & (raw > 0)].sum() + ), + "raw_employer_share_total": raw_universe_total, + "raw_anchor_universe_total": raw_anchor_total, "raw_employer_share_mean_per_positive_person": ( _weighted(weights, raw, raw_positive) / float(weights[raw_positive].sum()) if raw_positive.any() else 0.0 ), "raw_employer_share_band": list(_RAW_MEAN_BAND), + "raw_over_published_employer_mean": ( + _weighted(weights, raw, employer_pays) / published + if published + else float("nan") + ), "scale_factor": scale, "scale_factor_spread": ( float(np.nanmax(ratio) - np.nanmin(ratio)) if raw_positive.any() else 0.0 ), + "anchor_universe_employer_total": scale * raw_anchor_total, + "other_policyholder_employer_total": scale + * (raw_anchor_total - raw_universe_total), + "employed_share_of_anchor_universe": ( + raw_universe_total / raw_anchor_total if raw_anchor_total else float("nan") + ), "employer_premium_by_tier": by_tier, "employer_premium_by_sector": by_sector, - "employer_premium_outside_universe_rows": int( - (positive & ~codes.universe).sum() - ), + "employer_premium_outside_universe_rows": int((positive & ~universe).sum()), "employer_premium_where_employer_pays_none_rows": int( (positive & (codes.hipaid == _PAYS_NONE)).sum() ), "employer_premium_without_raw_share_rows": int( (positive & ~raw_positive).sum() ), - "pre_tax_ineligible_rows": int(((pre_tax > 0) & ~eligible).sum()), - "pre_tax_not_reported_premium_rows": int( - ((pre_tax > 0) & ~np.isclose(pre_tax, phip)).sum() + "meps_ic_private_active_employer_total": ( + meps_ic_private_active_employer_totals(load_meps_ic_esi_premium_cells()) ), - "pre_tax_eligible_weight": float(weights[eligible].sum()), - "pre_tax_selected_share_of_eligible": ( - float(weights[eligible & (pre_tax > 0)].sum()) - / float(weights[eligible].sum()) - if eligible.any() - else 0.0 - ), - "pre_tax_shares": _pre_tax_shares(cells), } return summary +def _missing_source_failure(missing: list[str], consequence: str) -> str: + return ( + f"assignment provenance incomplete: the person table lacks {missing}, so " + f"{consequence}. Rebuild the base with the meps_esi_premiums stage; a " + "release must carry the raw ASEC coverage columns the assignment read." + ) + + def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: - """Structure and scale-invariant signal of the two ESI premium inputs. + """Structure and scale-invariant signal of the ESI employer premium input. - Holds on any frame size (smoke builds included). On frames that still - carry the raw ASEC columns it also proves every positive value sits in the - employed-policyholder universe, equals one common multiple of its MEPS-IC - cell, and that pre-tax premiums are reported premiums of eligible people. + Holds on any frame size (smoke builds included). It proves every positive + value sits in the employed-policyholder universe and equals one common + multiple of its MEPS-IC cell. Those proofs read the raw ASEC columns, so a + frame without them fails: it cannot be certified. """ person = frame.table("person") @@ -964,6 +1024,13 @@ def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: details={"structural_error": str(exc)}, ) failures: list[str] = [] + if summary["source_columns_missing"]: + failures.append( + _missing_source_failure( + list(summary["source_columns_missing"]), + "the universe, payment-status and MEPS-IC cell checks did not run", + ) + ) for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: if person[column].dropna().nunique() < 2: failures.append(f"{column}: constant column — zero mass or no signal.") @@ -983,11 +1050,6 @@ def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: "employer_premium_without_raw_share_rows", "employer premium(s) with no MEPS-IC share", ), - ("pre_tax_ineligible_rows", "pre-tax premium(s) for ineligible people"), - ( - "pre_tax_not_reported_premium_rows", - "pre-tax premium(s) not equal to PHIP_VAL", - ), ): count = int(summary.get(key, 0)) if count: @@ -998,19 +1060,13 @@ def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: failures.append( f"employer premium positive share {share:.4f} outside [{low}, {high}]." ) - share = float(summary["pre_tax_premium_positive_share"]) - low, high = _PRE_TAX_POSITIVE_SHARE_BAND - if not low <= share <= high: - failures.append( - f"pre-tax premium positive share {share:.4f} outside [{low}, {high}]." - ) if "raw_employer_share_mean_per_positive_person" in summary: mean = float(summary["raw_employer_share_mean_per_positive_person"]) low, high = _RAW_MEAN_BAND if not low <= mean <= high: failures.append( f"raw MEPS-IC employer share per holder ${mean:,.0f} outside " - f"[${low:,.0f}, ${high:,.0f}]: the cell table is broken." + f"[${low:,.0f}, ${high:,.0f}]: the cell table's units are broken." ) # Relative to the factor: a float32 round trip leaves ~1e-7. spread = float(summary["scale_factor_spread"]) @@ -1028,14 +1084,19 @@ def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult: - """Hold a release's weighted ESI premium totals to their external anchors. - - Runs on the calibrated export (dense and sparse). Fails when either column - is absent or zero-mass, when the employer premium total leaves - ±``ANCHOR_RELATIVE_TOLERANCE`` of BEA NIPA 7.8 line 17 for ``time_period`` - or exceeds the retiree-inclusive NHE Table 24 employer contribution, or - when pre-tax premiums exceed NHE's employee contribution (which also - counts retiree- and COBRA-paid premiums). + """Hold a release's weighted ESI employer premium total to its anchor. + + Runs on the calibrated export (dense and sparse). The anchor, CMS NHE + Table 24, counts employer contributions for every policyholder, so the + gate compares the anchor-universe total: the column (employed + policyholders) plus the same scale factor applied to the other + policyholders' MEPS-IC shares, recomputed from the raw ASEC columns at the + frame's weights. It fails when the column is absent or zero-mass, when + the raw columns are absent, when that total leaves + ±``ANCHOR_RELATIVE_TOLERANCE`` of the anchor for ``time_period``, or when + the employed share of it leaves its band. BEA NIPA 7.8 line 17 and + MEPS-IC's own private-sector enrollment total are recorded beside the + verdict, not gated. """ person = frame.table("person") @@ -1051,59 +1112,92 @@ def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult failures=tuple(f"person column missing: {column}." for column in missing), details={"missing": missing}, ) - weights = np.asarray(frame.resolve_weights("person").values, dtype=np.float64) - employer = person[US_ESI_EMPLOYER_PREMIUM_COLUMN].to_numpy(dtype=np.float64) - pre_tax = person[US_ESI_PRE_TAX_PREMIUM_COLUMN].to_numpy(dtype=np.float64) - employer_total = _weighted(weights, employer) - pre_tax_total = _weighted(weights, pre_tax) + try: + summary = us_esi_premiums_summary(frame) + except SourceRuntimeError as exc: + return GateResult( + name="esi_premiums_anchor", + passed=False, + failures=(str(exc),), + details={"structural_error": str(exc)}, + ) + employer_total = float(summary["employer_premium_total"]) year = str(int(time_period)) failures: list[str] = [] details: dict[str, object] = { "time_period": int(time_period), "employer_premium_total": employer_total, - "pre_tax_premium_total": pre_tax_total, - "employer_premium_positive_persons": float(weights[employer > 0].sum()), + "employer_premium_positive_persons": summary[ + "employer_premium_positive_persons" + ], + "employer_premium_mean_per_positive_person": summary[ + "employer_premium_mean_per_positive_person" + ], "relative_tolerance": ANCHOR_RELATIVE_TOLERANCE, "anchor": dict(EMPLOYER_PREMIUM_ANCHOR), "cross_check": dict(EMPLOYER_PREMIUM_CROSS_CHECK), } - for column, total in ( - (US_ESI_EMPLOYER_PREMIUM_COLUMN, employer_total), - (US_ESI_PRE_TAX_PREMIUM_COLUMN, pre_tax_total), - ): - if not total > 0: - failures.append(f"{column}: zero weighted mass.") + if not employer_total > 0: + failures.append(f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: zero weighted mass.") anchor = EMPLOYER_PREMIUM_ANCHOR["values"].get(year) if anchor is None: - failures.append(f"no BEA NIPA 7.8 line 17 anchor for {year}.") + failures.append(f"no NHE Table 24 employer-contribution anchor for {year}.") + if summary["source_columns_missing"]: + failures.append( + _missing_source_failure( + list(summary["source_columns_missing"]), + "the anchor-universe total (every policyholder) cannot be computed", + ) + ) else: - relative = employer_total / float(anchor) - 1.0 - details |= {"anchor_value": float(anchor), "relative_error": relative} - if abs(relative) > ANCHOR_RELATIVE_TOLERANCE: - failures.append( - f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: weighted total " - f"${employer_total / 1e9:,.1f}B is {relative:+.1%} from BEA NIPA " - f"7.8 line 17 ${float(anchor) / 1e9:,.1f}B ({year}); tolerance " - f"±{ANCHOR_RELATIVE_TOLERANCE:.0%}." + universe_total = float(summary["anchor_universe_employer_total"]) + employed_share = float(summary["employed_share_of_anchor_universe"]) + details |= { + key: summary[key] + for key in ( + "anchor_universe_employer_total", + "other_policyholder_employer_total", + "employed_share_of_anchor_universe", + "scale_factor", + "weighted_policyholders", + "weighted_employed_policyholders", + "weighted_other_policyholders", + "employer_premium_by_sector", ) - nhe_employer = EMPLOYER_PREMIUM_CROSS_CHECK["employer_contribution"].get(year) - nhe_employee = EMPLOYER_PREMIUM_CROSS_CHECK["employee_contribution"].get(year) - if nhe_employer is not None: - details["cross_check_employer_ratio"] = employer_total / float(nhe_employer) - if employer_total > float(nhe_employer): + } + if not np.isfinite(universe_total): failures.append( - f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: weighted total " - f"${employer_total / 1e9:,.1f}B exceeds the retiree-inclusive NHE " - f"Table 24 employer contribution ${float(nhe_employer) / 1e9:,.1f}B." + "the anchor-universe employer total is not finite: no employed " + "policyholder carries a MEPS-IC share to recover the scale from." ) - if nhe_employee is not None: - details["cross_check_pre_tax_ratio"] = pre_tax_total / float(nhe_employee) - if pre_tax_total > float(nhe_employee): + elif anchor is not None: + relative = universe_total / float(anchor) - 1.0 + details |= {"anchor_value": float(anchor), "relative_error": relative} + if abs(relative) > ANCHOR_RELATIVE_TOLERANCE: + failures.append( + "employer premiums over every policyholder total " + f"${universe_total / 1e9:,.1f}B, {relative:+.1%} from NHE " + f"Table 24 ${float(anchor) / 1e9:,.1f}B ({year}); tolerance " + f"±{ANCHOR_RELATIVE_TOLERANCE:.0%}." + ) + low, high = _EMPLOYED_SHARE_BAND + details["employed_share_band"] = [low, high] + if not low <= employed_share <= high: failures.append( - f"{US_ESI_PRE_TAX_PREMIUM_COLUMN}: weighted total " - f"${pre_tax_total / 1e9:,.1f}B exceeds NHE Table 24's employee " - f"contribution ${float(nhe_employee) / 1e9:,.1f}B." + f"employed policyholders carry {employed_share:.4f} of the " + f"anchor-universe employer total, outside [{low}, {high}]." ) + bea = EMPLOYER_PREMIUM_CROSS_CHECK["values"].get(year) + if bea is not None and np.isfinite(universe_total): + details["cross_check_ratio"] = universe_total / float(bea) + active = dict(summary["meps_ic_private_active_employer_total"]) + if year in active: + private = float(dict(summary["employer_premium_by_sector"])["private"]) + details["private_active_cross_check"] = { + "meps_ic_private_active_employer_total": active[year], + "employer_premium_private_sector": private, + "ratio": private / active[year], + } return GateResult( name="esi_premiums_anchor", passed=not failures, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py index 36405d1b4f..646b04d968 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/operator_boundary.py @@ -44,6 +44,7 @@ from microcosm.build.us_runtime.energy_subsidy import ( US_ENERGY_SUBSIDY_OUTPUT_COLUMNS, ) +from microcosm.build.us_runtime.esi_premiums import US_ESI_PREMIUMS_OUTPUT_COLUMNS from microcosm.build.us_runtime.geography_ladder import US_GEOGRAPHY_LADDER_COLUMNS from microcosm.build.us_runtime.hours_worked import ( US_HOURS_WORKED_POOL_EXCLUDED_COLUMNS, @@ -320,6 +321,11 @@ "retirement_distributions": { "person": frozenset(US_RETIREMENT_DISTRIBUTION_OUTPUT_COLUMNS), }, + # Base-build only (microcosm#454): no pool operator produces these yet, so + # a raw source frame that already carries them was staged elsewhere. + "esi_premiums": { + "person": frozenset(US_ESI_PREMIUMS_OUTPUT_COLUMNS), + }, "immigration": { "person": frozenset(US_IMMIGRATION_OUTPUT_COLUMNS), }, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py index 40856b549c..38b4ecf8dc 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/release_input_coverage.py @@ -70,10 +70,7 @@ US_EDUCATOR_EXPENSE_OUTPUT_COLUMNS, ) from microcosm.build.us_runtime.energy_subsidy import US_ENERGY_SUBSIDY_OUTPUT_COLUMNS -from microcosm.build.us_runtime.esi_premiums import ( - US_ESI_EMPLOYER_PREMIUM_COLUMN, - US_ESI_PRE_TAX_PREMIUM_COLUMN, -) +from microcosm.build.us_runtime.esi_premiums import US_ESI_EMPLOYER_PREMIUM_COLUMN from microcosm.build.us_runtime.farm_business_income import ( US_FARM_BUSINESS_INCOME_OUTPUT_COLUMNS, ) @@ -193,10 +190,6 @@ # pinned Census ASEC person files; the certified default carries it # through the Build P source enrichment. *US_SPM_INDEPENDENCE_ROLE_OUTPUT_COLUMNS, - # PolicyEngine/microcosm#454: the employee section 125 premium leaf the - # meps_esi_premiums base-builder stage writes beside the restored - # employer premium. Absent, income-tax and FICA wages both overstate. - US_ESI_PRE_TAX_PREMIUM_COLUMN, } ) diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py b/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py index 6bff894ce9..b9247bc63c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/source_runtime.py @@ -44,7 +44,6 @@ ) from microcosm.build.us_runtime.esi_premiums import ( derive_us_employer_esi_premiums_from_manifest, - derive_us_pre_tax_health_insurance_premiums_from_manifest, ) from microcosm.build.us_runtime.hours_worked import ( derive_us_hours_worked_from_manifest, @@ -326,9 +325,6 @@ def us_source_operation_handlers() -> Mapping[str, SourceOperationHandler]: "derive_hours_worked": derive_us_hours_worked_from_manifest, "derive_medicare_take_up": derive_us_medicare_take_up_from_manifest, "derive_pregnancy": derive_us_pregnancy_from_manifest, - "derive_pre_tax_health_insurance_premiums": ( - derive_us_pre_tax_health_insurance_premiums_from_manifest - ), "derive_prior_year_income": derive_us_prior_year_income_from_manifest, "derive_relationship_inputs": derive_us_relationship_inputs_from_manifest, "derive_spm_independence_role": (derive_us_spm_independence_role_from_manifest), diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 571bbc9d0d..5dd6be077c 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,22 +82,22 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_252 - assert fields["authored_normative_field_count"] == 32_452 - assert fields["resolved_binding_field_count"] == 9_800 - assert fields["consumed_field_count"] == 42_252 + assert fields["configuration_field_count"] == 42_216 + assert fields["authored_normative_field_count"] == 32_436 + assert fields["resolved_binding_field_count"] == 9_780 + assert fields["consumed_field_count"] == 42_216 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_055, - "compiler_semantic": 27_740, + "legacy_behavior": 14_042, + "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_543, - "no_generation0_effect": 3_709, + "legacy_behavior": 38_530, + "no_generation0_effect": 3_686, } inventory = coverage_report["inventory_coverage"] @@ -108,7 +108,7 @@ def test_us_coverage_is_exact_complete_and_honest( assert inventory["counts"]["producer_inputs"] == 2_744 assert inventory["counts"]["ownership_rows"] == 18 assert inventory["counts"]["tail_control_fields"] == 934 - assert inventory["counts"]["seed_owner_bindings"] == 113 + assert inventory["counts"]["seed_owner_bindings"] == 112 @pytest.mark.parametrize( diff --git a/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py b/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py index a4647c46cb..fcca559c7a 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_us_esi_premiums_engine.py @@ -1,19 +1,16 @@ -"""Engine contract of the two ESI premium inputs (microcosm #454). +"""Engine contract of the ESI employer premium input (microcosm #454). ``docs/us-esi-employer-premiums.md`` and the ``meps_esi_premiums`` stage notes -say how PolicyEngine-US consumes each input. These tests compute it in the -locked engine, so an engine release that reroutes either input fails here -rather than silently changing what the stage's columns mean. +say how PolicyEngine-US consumes the input. These tests compute it in the +locked engine, so an engine release that reroutes it fails here rather than +silently changing what the stage's column means. """ from __future__ import annotations import pytest -from microcosm.build.us_runtime import ( - US_ESI_EMPLOYER_PREMIUM_COLUMN, - US_ESI_PRE_TAX_PREMIUM_COLUMN, -) +from microcosm.build.us_runtime import US_ESI_EMPLOYER_PREMIUM_COLUMN PERIOD = 2024 WAGES = 60_000.0 @@ -55,53 +52,24 @@ def baseline() -> dict[str, float]: return _totals() -def test_both_inputs_are_person_year_input_leaves() -> None: +def test_the_input_is_a_person_year_input_leaf() -> None: from policyengine_us import CountryTaxBenefitSystem - variables = CountryTaxBenefitSystem().variables - for name in (US_ESI_EMPLOYER_PREMIUM_COLUMN, US_ESI_PRE_TAX_PREMIUM_COLUMN): - variable = variables[name] - assert variable.is_input_variable(), name - assert variable.entity.key == "person", name - assert str(variable.definition_period).lower() == "year", name + variable = CountryTaxBenefitSystem().variables[US_ESI_EMPLOYER_PREMIUM_COLUMN] + assert variable.is_input_variable() + assert variable.entity.key == "person" + assert str(variable.definition_period).lower() == "year" -def test_the_engine_parameter_lists_name_the_inputs() -> None: +def test_the_engine_parameter_list_names_the_input() -> None: from policyengine_us import CountryTaxBenefitSystem parameters = CountryTaxBenefitSystem().parameters - instant = f"{PERIOD}-01-01" assert US_ESI_EMPLOYER_PREMIUM_COLUMN in ( - parameters.gov.household.cbo_market_income_additions(instant) - ) - assert US_ESI_PRE_TAX_PREMIUM_COLUMN in ( - parameters.gov.irs.gross_income.pre_tax_contributions(instant) - ) - assert US_ESI_PRE_TAX_PREMIUM_COLUMN in ( - parameters.gov.irs.gross_income.fica_pre_tax_contributions(instant) + parameters.gov.household.cbo_market_income_additions(f"{PERIOD}-01-01") ) -def test_a_pre_tax_premium_lowers_income_tax_and_fica_wages_dollar_for_dollar( - baseline, -) -> None: - premium = 2_000.0 - reduced = _totals(**{US_ESI_PRE_TAX_PREMIUM_COLUMN: premium}) - - for measure in ( - "irs_employment_income", - "payroll_tax_gross_wages", - "adjusted_gross_income", - ): - assert reduced[measure] - baseline[measure] == pytest.approx(-premium), measure - assert baseline["irs_employment_income"] == pytest.approx(WAGES) - assert reduced["income_tax"] < baseline["income_tax"] - # 6.2% employee OASDI on the premium no longer in FICA wages. - assert reduced["employee_social_security_tax"] - baseline[ - "employee_social_security_tax" - ] == pytest.approx(-0.062 * premium) - - def test_an_employer_premium_moves_only_cbo_household_market_income(baseline) -> None: premium = 10_000.0 with_premium = _totals(**{US_ESI_EMPLOYER_PREMIUM_COLUMN: premium}) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py index e0ed3f5e12..b025205ae5 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_compiler_ir.py @@ -107,9 +107,9 @@ def test_us_seed_stream_map_is_complete_and_owner_typed( ) -> None: seed_map = compiled_us.seed_stream_map assert seed_map.protocol_id == "legacy-v1" - assert len(seed_map.sites) == 54 - assert len(seed_map.owners) == 55 - assert sum(len(site.owners) for site in seed_map.sites) == 113 + assert len(seed_map.sites) == 53 + assert len(seed_map.owners) == 54 + assert sum(len(site.owners) for site in seed_map.sites) == 112 assert {site.id for site in seed_map.sites} == { site.id for site in resolved_us.seed_protocol.sites } @@ -244,5 +244,5 @@ def test_bundle_without_imputation_compiles_empty_graph( assert compiled.producer_graph.nodes == () assert compiled.stage_dag.nodes == () assert compiled.nodes == () - assert len(compiled.seed_stream_map.sites) == 54 + assert len(compiled.seed_stream_map.sites) == 53 assert compiled.seed_stream_map.owners == () diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index 2e7a1cd068..4babaf7512 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_252 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_216 assert field_ledger.source_counts == { - "authored": 32_452, - "resolved_bindings": 9_800, + "authored": 32_436, + "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_055, - "compiler_semantic": 27_740, + "legacy_behavior": 14_042, + "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_543, - "no_generation0_effect": 3_709, + "legacy_behavior": 38_530, + "no_generation0_effect": 3_686, } - assert len({field.pointer for field in field_ledger.fields}) == 42_252 + assert len({field.pointer for field in field_ledger.fields}) == 42_216 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py index 73ce670319..fce631f3a5 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_identity_contracts.py @@ -118,13 +118,13 @@ def test_identity_contract_objects_are_closed_world( load_schema_registry().validate(mutated, "spine.schema.json") -def test_all_54_seed_sites_resolve_to_typed_real_owners( +def test_all_53_seed_sites_resolve_to_typed_real_owners( identity_documents: tuple[dict[str, object], list[dict[str, object]]], ) -> None: spine, source_stages = identity_documents bindings = _resolve(spine, source_stages) - assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 54 + assert len(bindings) == len(LEGACY_V1_PROTOCOL.sites) == 53 assert [binding.site for binding in bindings] == [ site.id for site in LEGACY_V1_PROTOCOL.sites ] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py index 7be042baf6..10b377b286 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_loader.py @@ -12,7 +12,7 @@ def test_semantic_hash_has_golden_vector_and_surface_separation(tmp_path) -> Non # Pin the domain separator, normalization rules, schema-set receipt, and # exact normative projection as one reviewable golden vector. assert first.spec_sha256 == ( - "a48d950ffc0079ee67a4994e0b8df8d25a9ad4d660dc7e9ef0d3337df55e49d9" + "89839365bc078a8206a1ddffb73885ff71e0decf4eba19c86e5ed2a930fde0ae" ) second_root = _rich_minimal(tmp_path / "xy", note="second", store="local:b") diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py index 8125365db7..78df801e0d 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_plan_lock.py @@ -42,7 +42,7 @@ def test_plan_lock_is_complete_and_closed_world(compiled_us: CompiledSpecIR) -> assert payload["spec_binding"]["attestation"] == "mirror-attested" assert len(payload["producer_graph"]["nodes"]) == 38 assert len(payload["producer_graph"]["authored"]["ownership_matrix"]) == 18 - assert len(payload["seed_stream_map"]["sites"]) == 54 + assert len(payload["seed_stream_map"]["sites"]) == 53 assert len(payload["nodes"]) == 38 mutated = copy.deepcopy(payload) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py index 7b1e53311e..bba3a43b43 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_seeds.py @@ -50,7 +50,6 @@ "legacy_puma_ladder", "medicaid_take_up_assignment", "other_health_insurance_training_cap", - "pre_tax_health_premium_assignment", "pregnancy_assignment", "primary_qrf_fit_draw", "prior_year_income_training_cap", @@ -137,7 +136,6 @@ ), "snap_take_up_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/snap_take_up.py", "pregnancy_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/pregnancy.py", - "pre_tax_health_premium_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py", "wic_claim_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/wic_claim.py", "snap_discretionary_exemption_assignment": "packages/microcosm-build/src/microcosm/build/us_runtime/snap_discretionary_exemption.py", **dict.fromkeys( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py index e4c841adb2..e7a20bff8a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_acs_local_release_tool.py @@ -2096,3 +2096,49 @@ def test_concentration_limitation_is_true_to_the_penalty_solved() -> None: assert "'chi_square'" in penalized["reason"] assert "'softmax'" in penalized["reason"] assert "l2_lambda=0 " not in penalized["reason"] + + +def test_a_null_esi_premium_input_is_refused_before_any_default_fill(tmp_path) -> None: + """microcosm#454: an ACS-spine null in the ESI premium input is never a zero. + + The ``meps_esi_premiums`` stage runs on the ASEC donor rows only. A pooled + frame therefore carries the input as NaN on its ACS rows, and the + reviewed-null register would default-fill it to zero for half the + population while input coverage reads green. The refusal comes first: it + needs neither the engine nor the register file. + """ + + from microcosm.build.source_runtime import SourceRuntimeError + from microcosm.frame import US_SCHEMA, Frame, WeightKind, Weights + + module = _load_tool_module() + ids = np.arange(1, 5) + person = pd.DataFrame( + { + "person_id": ids, + "person_household_id": ids, + "person_tax_unit_id": ids, + "person_spm_unit_id": ids, + "person_family_id": ids, + "person_marital_unit_id": ids, + # Two ASEC donor rows, two ACS rows the stage never saw. + "employer_sponsored_insurance_premiums": [9_000.0, 0.0, np.nan, np.nan], + } + ) + frame = Frame( + { + "person": person, + "household": pd.DataFrame({"household_id": ids}), + "tax_unit": pd.DataFrame({"tax_unit_id": ids}), + "spm_unit": pd.DataFrame({"spm_unit_id": ids}), + "family": pd.DataFrame({"family_id": ids}), + "marital_unit": pd.DataFrame({"marital_unit_id": ids}), + }, + US_SCHEMA, + {"household": Weights(values=np.ones(4), kind=WeightKind.DESIGN)}, + ) + absent_register = tmp_path / "no_such_summary.json" + with pytest.raises(SourceRuntimeError, match="never filled") as error: + module.fill_reviewed_nulls(frame, absent_register) + assert "ACS local release" in str(error.value) + assert "'employer_sponsored_insurance_premiums': 2" in str(error.value) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index c5203a6203..4d4d52c0b4 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91", + "sha256": "e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py index cb48b2ee72..5e7ec72b35 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py @@ -2,17 +2,23 @@ Invariants the stage and its gates must hold for every valid input: -* **Conservation**: the weighted employer premium total equals the BEA NIPA - 7.8 line 17 anchor of the build year, whatever the frame. -* **Bounds**: both outputs are finite and nonnegative; a pre-tax premium is - either zero or the person's reported premium. +* **Conservation**: the weighted employer share over every policyholder (the + column plus the other policyholders' share at the same scale factor) equals + the CMS NHE Table 24 anchor of the build year, whatever the frame. The + column total is the anchor times the employed policyholders' share of the + weighted raw mass, so it equals the anchor exactly when no one outside the + column carries an employer share. +* **Monotonicity**: pricing a policyholder outside the column never raises a + worker's premium. +* **Bounds**: the output is finite and nonnegative. * **Structural zeros**: no employer premium outside employed policyholders - with an employer, nor where the employer pays none; no pre-tax premium - outside eligible workers. + with an employer, nor where the employer pays none. * **Proportionality**: every employer premium is the same multiple of its MEPS-IC cell share. -* **Determinism and order invariance**: values depend on the person, the - frame's weighted raw mass and the seed, never on row order; support clones +* **Cell reconciliation**: a cohort with MEPS-IC's own pays-all / pays-some + mix reproduces the cell's published employer mean. +* **Determinism and order invariance**: values depend on the person and the + frame's weighted raw mass, never on the seed or row order; support clones agree. * **Weight homogeneity**: scaling every weight by ``c`` leaves the total at the anchor and scales each person's value by ``1 / c``. @@ -35,17 +41,14 @@ from hypothesis import strategies as st from microcosm.build.source_runtime import SourceRuntimeError -from microcosm.build.stochastic_assignment import stable_identity_uniforms from microcosm.build.us_runtime import ( US_ESI_EMPLOYER_PREMIUM_COLUMN, - US_ESI_PRE_TAX_PREMIUM_COLUMN, US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, US_ESI_PREMIUMS_OUTPUT_COLUMNS, US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS, US_ESI_PREMIUMS_STAGE_NAME, US_SOURCE_MANIFEST, derive_us_employer_esi_premiums_from_manifest, - derive_us_pre_tax_health_insurance_premiums_from_manifest, load_meps_ic_esi_premium_cells, us_esi_premiums_anchor_gate, us_esi_premiums_signal_gate, @@ -57,6 +60,10 @@ from microcosm.build.us_runtime.asec_census_person_columns import ( ASEC_CENSUS_PERSON_COLUMNS, ) +from microcosm.build.us_runtime.esi_premiums import ( + meps_ic_private_active_employer_totals, + refuse_unassigned_us_esi_premiums, +) from microcosm.build.us_runtime.release_input_coverage import ( us_release_input_coverage_required_columns, us_release_input_coverage_reviewed_exclusions, @@ -67,15 +74,17 @@ from test_support.paths import paths_for TIME_PERIOD = 2024 -ANCHOR = 977_034e6 +#: CMS NHE Table 24 employer contribution, CY2024: the scale anchor. +ANCHOR = 1_047_000e6 +#: BEA NIPA Table 7.8 line 17, CY2024 (2026-09-30 vintage): the cross-check. +BEA = 977_034e6 EMPLOYER = US_ESI_EMPLOYER_PREMIUM_COLUMN -PRE_TAX = US_ESI_PRE_TAX_PREMIUM_COLUMN _REPOSITORY_ROOT = paths_for("microcosm-build").repository _CELLS = load_meps_ic_esi_premium_cells() _STATES = sorted(int(state) for state in _CELLS["state_census_division"]) #: An employed private-sector self-only policyholder in a large California -#: firm whose employer pays some of the premium, with wages and a premium. +#: firm whose employer pays some of the premium. _BASE_ROW = { "NOW_OWNGRP": 1, "NOW_HIPAID": 2, @@ -84,8 +93,6 @@ "PEMLR": 1, "NOEMP": 6, "PEIO1COW": 4, - "PHIP_VAL": 2_000, - "WSAL_VAL": 60_000, "state_fips": 6, } _NO_COVERAGE = {"NOW_OWNGRP": 2, "NOW_HIPAID": 0, "NOW_GRPFTYP": 0, "NOW_GRPFTYP2": 0} @@ -152,6 +159,24 @@ def _private(tier: str, measure: str, state: int, size: str) -> float: return _CELLS["private_state_2025"][tier][measure]["rows"][f"{state:02d}"][size] +def _no_contribution(tier: str, size: str) -> float: + return _CELLS["no_contribution_share_2025"][tier][size] / 100.0 + + +def _column_total(frame: Frame) -> float: + return float(_weights(frame) @ _values(frame, EMPLOYER)) + + +def _reweighted(frame: Frame, weights: np.ndarray) -> Frame: + """The same people at different household weights (a calibrated export).""" + + return Frame( + {entity: frame.table(entity) for entity in frame.entities}, + US_SCHEMA, + {"household": Weights(values=np.asarray(weights), kind=WeightKind.DESIGN)}, + ) + + def _realistic_rows(count: int = 400) -> list[dict]: """A mix whose weighted shares sit inside the signal-gate bands.""" @@ -167,13 +192,12 @@ def _realistic_rows(count: int = 400) -> list[dict]: "NOW_GRPFTYP": 1 if tier in (1, 2) else 2, "NOEMP": (index // 7) % 7, "PEIO1COW": (1, 2, 3, 4, 5, 6)[(index // 3) % 6], - "PHIP_VAL": 1_000 + 37 * index, "state_fips": state, } ) elif kind == 4: # employer pays all - rows.append({"NOW_HIPAID": 1, "PHIP_VAL": 0, "state_fips": state}) - elif kind == 5: # retiree policyholder: no wages concept + rows.append({"NOW_HIPAID": 1, "state_fips": state}) + elif kind == 5: # retiree policyholder: priced, outside the column rows.append({"PEMLR": 5, "PEIO1COW": 0, "NOEMP": 0, "state_fips": state}) else: # no employment-based coverage of their own rows.append( @@ -194,11 +218,11 @@ def test_stage_declares_both_outputs_and_the_reviewed_operations(self) -> None: spec = us_esi_premiums_stage_spec() assert spec.stage == US_ESI_PREMIUMS_STAGE_NAME == "meps_esi_premiums" assert tuple(spec.outputs) == US_ESI_PREMIUMS_OUTPUT_COLUMNS - assert US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS == (EMPLOYER, PRE_TAX) + assert US_ESI_PREMIUMS_OUTPUT_COLUMNS == (EMPLOYER,) + assert US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS == (EMPLOYER,) assert [operation.kind for operation in spec.operations] == [ "read_table", "derive_employer_sponsored_insurance_premiums", - "derive_pre_tax_health_insurance_premiums", ] assert spec is not None and US_SOURCE_MANIFEST.stage_map()[spec.stage] @@ -208,34 +232,35 @@ def test_handlers_are_registered(self) -> None: handlers["derive_employer_sponsored_insurance_premiums"] is derive_us_employer_esi_premiums_from_manifest ) - assert ( - handlers["derive_pre_tax_health_insurance_premiums"] - is derive_us_pre_tax_health_insurance_premiums_from_manifest - ) + # The pre-tax employee premium is not this stage's: the engine treats + # it as disjoint from the reported premium (policyengine-us#10046). + assert "derive_pre_tax_health_insurance_premiums" not in handlers def test_manifest_pins_the_cells_the_anchor_and_every_source_pdf(self) -> None: spec = us_esi_premiums_stage_spec() employer = spec.operations[1].parameters assert employer["cells_sha256"] == esi._CELLS_SHA256 - assert employer["anchor"]["series"] == "B4923C" + assert employer["anchor"]["table"].startswith("NHE Table 24") + assert employer["anchor"]["row"] == esi.EMPLOYER_PREMIUM_ANCHOR["row"] assert employer["anchor"]["values"]["2024"] == ANCHOR + assert employer["anchor_universe"] == "NOW_OWNGRP == 1" + assert "PEMLR in (1, 2)" in employer["universe"] + # A zip member is addressed as "!". pinned = { - artifact["locator"]: artifact["sha256"] + artifact["locator"].split("!")[0]: artifact["sha256"] for artifact in spec.artifacts if artifact.get("sha256") } for source in _CELLS["sources"].values(): assert pinned[source["url"]] == source["sha256"] - assert ( - pinned[esi.EMPLOYER_PREMIUM_ANCHOR["source"]] - == (esi.EMPLOYER_PREMIUM_ANCHOR["sha256"]) - ) + for pin in (esi.EMPLOYER_PREMIUM_ANCHOR, esi.EMPLOYER_PREMIUM_CROSS_CHECK): + assert pinned[pin["source"]] == pin["sha256"] def test_a_drifted_manifest_operation_is_refused(self, monkeypatch) -> None: monkeypatch.setattr( esi, - "_PRE_TAX_PARAMETERS", - {**esi._PRE_TAX_PARAMETERS, "employer_payment_codes": [2]}, + "_EMPLOYER_PREMIUM_PARAMETERS", + {**esi._EMPLOYER_PREMIUM_PARAMETERS, "tier_column": "NOW_GRPFTYP"}, ) with pytest.raises(ValueError, match="drifted"): us_esi_premiums_stage_spec() @@ -248,8 +273,6 @@ def test_the_restored_census_columns_are_the_ones_the_stage_requires(self) -> No assert restored["NOEMP"].domain == esi._CODE_DOMAINS["NOEMP"] assert set(restored) >= set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) - { "PEIO1COW", - "PHIP_VAL", - "WSAL_VAL", "state_fips", } @@ -316,6 +339,9 @@ def test_every_resolved_cell_leaves_a_positive_employer_share(self) -> None: _CELLS, tier, "employee_contribution", state, size ) assert 0 < paid < premium < 60_000 + # The contribution of an employee who pays one still + # leaves the employer a share. + assert paid / (1 - _no_contribution(tier, size)) < premium for division in set(_CELLS["state_census_division"].values()): for column in ("state", "all_state_and_local"): premium = esi._government_cell( @@ -325,33 +351,153 @@ def test_every_resolved_cell_leaves_a_positive_employer_share(self) -> None: _CELLS, tier, "employee_contribution", division, column ) assert 0 < paid < premium < 60_000 - - def test_pre_tax_shares_are_offer_rate_ratios(self) -> None: - shares = esi._pre_tax_shares(_CELLS) - assert shares == pytest.approx( - {"lt50": 15.9 / 32.1, "50plus": 87.8 / 94.8, "total": 35.5 / 49.2} + assert paid / (1 - _no_contribution(tier, "50plus")) < premium + + def test_no_contribution_shares_are_the_published_national_rows(self) -> None: + # MEPS-IC 2025 Tables II.C.4.a, II.D.4.a and II.E.4.a, United States. + published = { + "single": {"total": 13.0, "lt50": 35.5, "50plus": 8.7}, + "family": {"total": 7.7, "lt50": 28.6, "50plus": 4.6}, + "employee_plus_one": {"total": 6.0, "lt50": 23.1, "50plus": 3.6}, + } + for tier, row in published.items(): + table = _CELLS["no_contribution_share_2025"][tier] + assert {size: table[size] for size in row} == row + assert "required no" in table["title"] + + def test_private_enrollment_rows_price_the_active_employee_total(self) -> None: + totals = meps_ic_private_active_employer_totals(_CELLS) + rows = _CELLS["private_enrollment_national"]["2024"] + assert rows["employees"]["total"] == 139_441_077 + assert rows["offer_percent"]["total"] == 85.1 + assert rows["enrolled_percent"]["total"] == 55.3 + enrolled = 139_441_077 * 0.851 * 0.553 + by_hand = enrolled * ( + 0.587 * (8_486 - 1_789) + + 0.237 * (24_540 - 7_216) + + 0.176 * (16_931 - 4_707) ) - assert all(0 < share < 1 for share in shares.values()) + assert totals["2024"] == pytest.approx(by_hand, rel=1e-12) + assert totals["2024"] == pytest.approx(668.6e9, rel=5e-4) + assert totals["2025"] == pytest.approx(714.4e9, rel=5e-4) + for year in ("2024", "2025"): + shares = _CELLS["private_enrollment_national"][year]["tier_share_percent"] + assert sum(shares[tier]["total"] for tier in _CELLS["tiers"]) == ( + pytest.approx(100.0) + ) # --- the employer premium ------------------------------------------------------- class TestEmployerPremium: - def test_weighted_total_is_the_bea_anchor(self) -> None: + def test_weighted_total_over_every_policyholder_is_the_nhe_anchor(self) -> None: result = _run(_realistic_rows(), weights=np.linspace(50, 5_000, 400)) - total = _weights(result) @ _values(result, EMPLOYER) - assert total == pytest.approx(ANCHOR, rel=1e-12) + summary = us_esi_premiums_summary(result) + assert summary["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + # The retiree policyholders are priced, so the column is the employed + # policyholders' part of the anchor, not all of it. + share = summary["employed_share_of_anchor_universe"] + assert 0 < share < 1 + assert _column_total(result) == pytest.approx(ANCHOR * share, rel=1e-12) + assert summary["other_policyholder_employer_total"] == pytest.approx( + ANCHOR * (1 - share), rel=1e-9 + ) + + def test_without_other_policyholders_the_column_is_the_whole_anchor(self) -> None: + rows = [{"NOW_HIPAID": 1}, {"NOW_HIPAID": 2}, dict(_NO_COVERAGE)] + assert _column_total(_run(rows)) == pytest.approx(ANCHOR, rel=1e-12) + + def test_a_priced_retiree_takes_their_share_out_of_the_column(self) -> None: + # A worker and a retiree whose former employer pays the whole premium. + # The retiree has no employer class, so they take California's + # all-sizes private cell; the worker takes the 50-or-more cell. + retiree = {"NOW_HIPAID": 1, "PEMLR": 5, "PEIO1COW": 0, "NOEMP": 0} + alone = _values(_run([{"NOW_HIPAID": 1}]), EMPLOYER) + result = _run([{"NOW_HIPAID": 1}, retiree]) + together = _values(result, EMPLOYER) + worker = _private("single", "premium", 6, "50plus") + retired = _private("single", "premium", 6, "total") + assert alone[0] == pytest.approx(ANCHOR) + assert together[1] == 0 + assert together[0] == pytest.approx(ANCHOR * worker / (worker + retired)) + summary = us_esi_premiums_summary(result) + assert summary["other_policyholder_employer_total"] == pytest.approx( + ANCHOR * retired / (worker + retired) + ) + assert summary["weighted_other_policyholders_with_employer_share"] == 1 + + def test_a_non_employed_policyholder_keeps_the_cell_of_a_reported_employer( + self, + ) -> None: + # Unemployed, last job with a State government in Texas, small NOEMP: + # the State-government division cell, whatever the firm size. + former = {"NOW_HIPAID": 1, "PEMLR": 4, "PEIO1COW": 2, "NOEMP": 1} + result = _run([{"NOW_HIPAID": 1}, {**former, "state_fips": 48}]) + summary = us_esi_premiums_summary(result) + published = _CELLS["public_division_2024"]["single"]["premium"]["rows"][ + "West South Central" + ]["state"] + aged = published * 9_025 / 8_486 + assert summary["other_policyholder_employer_total"] / _column_total( + result + ) == pytest.approx(aged / _private("single", "premium", 6, "50plus")) def test_payment_status_sets_the_share_of_the_cell(self) -> None: - rows = [{"NOW_HIPAID": code, "PHIP_VAL": 500} for code in (1, 2, 3)] + rows = [{"NOW_HIPAID": code} for code in (1, 2, 3)] employer = _values(_run(rows), EMPLOYER) premium = _private("single", "premium", 6, "50plus") paid = _private("single", "employee_contribution", 6, "50plus") + # The published average contribution counts the 8.7% who pay nothing. + paying = paid / (1 - 0.087) + assert _no_contribution("single", "50plus") == 0.087 assert employer[2] == 0 - assert employer[0] / employer[1] == pytest.approx(premium / (premium - paid)) + assert employer[0] / employer[1] == pytest.approx(premium / (premium - paying)) # One scale factor: both are the same multiple of their raw share. - assert employer[0] / premium == pytest.approx(employer[1] / (premium - paid)) + assert employer[0] / premium == pytest.approx(employer[1] / (premium - paying)) + + @pytest.mark.parametrize("tier_code", [1, 2, 3]) + @pytest.mark.parametrize(("noemp", "size"), [(1, "lt50"), (4, "50plus")]) + @pytest.mark.parametrize("state", [6, 36, 48]) + def test_the_published_mix_reproduces_the_published_employer_mean( + self, tier_code, noemp, size, state + ) -> None: + tier = esi._TIER_BY_CODE[tier_code] + share = _no_contribution(tier, size) + cell = { + "NOW_GRPFTYP2": tier_code, + "NOW_GRPFTYP": 1 if tier_code in (1, 2) else 2, + "NOEMP": noemp, + "state_fips": state, + } + # MEPS-IC's own mix: `share` of enrollees pay nothing, the rest pay. + result = _run( + [{**cell, "NOW_HIPAID": 1}, {**cell, "NOW_HIPAID": 2}], + weights=[share, 1 - share], + ) + summary = us_esi_premiums_summary(result) + assert summary["raw_over_published_employer_mean"] == pytest.approx(1.0) + published = _private(tier, "premium", state, size) - esi._private_cell( + _CELLS, tier, "employee_contribution", f"{state:02d}", size + ) + modeled = _column_total(result) / summary["scale_factor"] + assert modeled == pytest.approx(published) + + def test_the_unconditional_contribution_would_overstate_the_cell_mean(self) -> None: + # The national under-50 single-coverage cell: premium $9,034, average + # contribution $1,924, 35.5% of enrollees pay nothing. Subtracting the + # average from every payer gives $7,793, above the published $7,110. + rows = _CELLS["private_state_2025"]["single"] + premium = rows["premium"]["rows"]["US"]["lt50"] + paid = rows["employee_contribution"]["rows"]["US"]["lt50"] + share = _no_contribution("single", "lt50") + assert (premium, paid, share) == (9_034, 1_924, 0.355) + unconditional = share * premium + (1 - share) * (premium - paid) + conditional = share * premium + (1 - share) * (premium - paid / (1 - share)) + assert unconditional == pytest.approx(7_793.02) + assert conditional == pytest.approx(premium - paid) == pytest.approx(7_110) def test_only_employed_policyholders_with_an_employer_carry_a_premium(self) -> None: rows = [ @@ -435,9 +581,13 @@ def test_a_suppressed_cell_takes_the_national_cell_at_the_state_level(self) -> N / contributions["US"]["total"] ) premium = _private("family", "premium", 2, "lt50") - assert employer[1] / employer[0] == pytest.approx( - (premium - fallback) / premium - ) + paying = fallback / (1 - _no_contribution("family", "lt50")) + assert employer[1] / employer[0] == pytest.approx((premium - paying) / premium) + + def test_the_assignment_does_not_depend_on_the_seed(self) -> None: + rows = _realistic_rows(120) + first, second = _run(rows, seed=1), _run(rows, seed=2) + assert np.array_equal(_values(first, EMPLOYER), _values(second, EMPLOYER)) def test_state_may_already_sit_on_the_person_table(self) -> None: rows = _realistic_rows(60) @@ -451,10 +601,10 @@ def test_state_may_already_sit_on_the_person_table(self) -> None: _values(on_household, EMPLOYER), _values(on_person, EMPLOYER) ) - @pytest.mark.parametrize("year", [2023, 2025]) + @pytest.mark.parametrize("year", [2023, 2024]) def test_each_pinned_year_scales_to_its_own_anchor(self, year) -> None: result = _run(_realistic_rows(60), time_period=year) - total = _weights(result) @ _values(result, EMPLOYER) + total = us_esi_premiums_summary(result)["anchor_universe_employer_total"] assert total == pytest.approx( esi.EMPLOYER_PREMIUM_ANCHOR["values"][str(year)], rel=1e-12 ) @@ -501,67 +651,31 @@ def test_the_census_universe_relations_are_enforced(self, row, violation) -> Non with pytest.raises(SourceRuntimeError, match=violation): _run([{}, row]) - def test_a_frame_with_no_employer_paid_mass_cannot_be_scaled(self) -> None: + @pytest.mark.parametrize( + "other", + [ + dict(_NO_COVERAGE), + # A priced retiree does not give the column any mass to scale. + {"NOW_HIPAID": 1, "PEMLR": 5, "PEIO1COW": 0}, + ], + ) + def test_a_frame_with_no_employer_paid_mass_cannot_be_scaled(self, other) -> None: with pytest.raises(SourceRuntimeError, match="no weighted employer-paid"): - _run([{"NOW_HIPAID": 3}, dict(_NO_COVERAGE)]) + _run([{"NOW_HIPAID": 3}, other]) - def test_an_unpinned_build_year_is_refused(self) -> None: - with pytest.raises(SourceRuntimeError, match="no BEA anchor for 2030"): - _run([{}], time_period=2030) + @pytest.mark.parametrize("year", [2025, 2030]) + def test_an_unpinned_build_year_is_refused(self, year) -> None: + # NHE Table 24 ends at CY2024; BEA's 2025 value does not stand in. + with pytest.raises( + SourceRuntimeError, match=f"no NHE Table 24 anchor for {year}" + ): + _run([{}], time_period=year) def test_an_unknown_state_is_refused(self) -> None: with pytest.raises(SourceRuntimeError, match="State FIPS 72"): _run([{}, {"state_fips": 72}]) -# --- the pre-tax employee premium ------------------------------------------------ - - -class TestPreTaxPremium: - def test_only_eligible_workers_can_pay_pre_tax(self) -> None: - eligible = [{"person_id": index + 1} for index in range(300)] - ineligible = [ - {"NOW_HIPAID": 1}, # the employer pays everything - {"WSAL_VAL": 0}, # no wages to deduct from - {"PHIP_VAL": 0}, # no premium reported - {"PEMLR": 5, "PEIO1COW": 0}, # not employed - {"PEIO1COW": 7}, # no employer - dict(_NO_COVERAGE), # not a policyholder - ] - result = _run(eligible + ineligible) - pre_tax = _values(result, PRE_TAX) - assert (pre_tax[300:] == 0).all() - assert set(np.unique(pre_tax[:300])) == {0.0, 2_000.0} - - @pytest.mark.parametrize( - ("noemp", "size"), [(0, "total"), (1, "lt50"), (2, "lt50"), (6, "50plus")] - ) - def test_selection_rate_is_the_firm_size_offer_ratio(self, noemp, size) -> None: - # Both payment statuses that leave the worker a share to pay. - rows = [{"NOEMP": noemp, "NOW_HIPAID": 2 + index % 2} for index in range(6_000)] - pre_tax = _values(_run(rows, seed=11), PRE_TAX) - share = esi._pre_tax_shares(_CELLS)[size] - # Six thousand Bernoulli draws: five standard errors is under 3.3 points. - assert (pre_tax > 0).mean() == pytest.approx(share, abs=0.033) - - def test_draws_are_the_shared_identity_keyed_uniforms(self) -> None: - rows = [{"source_household_id": 900 + index} for index in range(50)] - pre_tax = _values(_run(rows, seed=5), PRE_TAX) - draws = stable_identity_uniforms( - [f"2024:{900 + index}:1" for index in range(50)], - seed=5, - salt=PRE_TAX, - ) - expected = np.where(draws < esi._pre_tax_shares(_CELLS)["50plus"], 2_000.0, 0.0) - assert np.array_equal(pre_tax, expected) - - def test_the_seed_changes_who_is_selected_not_the_employer_premium(self) -> None: - rows = [{"NOEMP": 1} for _ in range(400)] - first, second = _run(rows, seed=1), _run(rows, seed=2) - assert not np.array_equal(_values(first, PRE_TAX), _values(second, PRE_TAX)) - assert np.array_equal(_values(first, EMPLOYER), _values(second, EMPLOYER)) - - # --- idempotence and support clones ----------------------------------------------- @@ -636,18 +750,26 @@ def test_passes_on_a_staged_frame_and_reports_the_lineage(self) -> None: gate = us_esi_premiums_signal_gate(self._staged()) assert gate.passed, gate.failures details = gate.details - assert details["employer_premium_total"] == pytest.approx(ANCHOR, rel=1e-12) + column = details["employer_premium_total"] + assert details["source_columns_missing"] == [] + assert details["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + assert column == pytest.approx( + ANCHOR * details["employed_share_of_anchor_universe"], rel=1e-12 + ) assert details["scale_factor"] == pytest.approx( - ANCHOR / details["raw_employer_share_total"] + ANCHOR / details["raw_anchor_universe_total"] ) assert details["scale_factor_spread"] <= 1e-9 * details["scale_factor"] assert details["cells_sha256"] == esi._CELLS_SHA256 - assert details["anchor"]["series"] == "B4923C" + assert details["anchor"]["row"].startswith("Employer Contribution") + assert details["cross_check"]["series"] == "B4923C" assert sum(details["employer_premium_by_tier"].values()) == pytest.approx( - ANCHOR + column ) assert sum(details["employer_premium_by_sector"].values()) == pytest.approx( - ANCHOR + column ) json.dumps(dict(details)) # the base summary serializes it @@ -696,24 +818,10 @@ def mutate(person): def test_a_negative_value_fails(self) -> None: self._fails_with( - lambda person: person.__setitem__(PRE_TAX, -person[PRE_TAX]), + lambda person: person.__setitem__(EMPLOYER, -person[EMPLOYER]), "negative value", ) - def test_a_pre_tax_premium_for_an_ineligible_person_fails(self) -> None: - def mutate(person): - outsider = int(np.flatnonzero(person["NOW_OWNGRP"].to_numpy() == 2)[0]) - person.loc[outsider, PRE_TAX] = float(person.loc[outsider, "PHIP_VAL"]) - - self._fails_with(mutate, "ineligible people") - - def test_a_pre_tax_premium_that_is_not_the_reported_premium_fails(self) -> None: - def mutate(person): - payer = int(np.flatnonzero(person[PRE_TAX].to_numpy() > 0)[0]) - person.loc[payer, PRE_TAX] += 1.0 - - self._fails_with(mutate, "not equal to PHIP_VAL") - def test_clone_disagreement_fails(self) -> None: cloned = _cloned(self._staged()) person = cloned.table("person") @@ -726,68 +834,96 @@ def test_an_implausible_positive_share_fails(self) -> None: gate = us_esi_premiums_signal_gate(_run([{} for _ in range(50)])) assert any("positive share" in failure for failure in gate.failures) - def test_a_broken_cell_table_cannot_hide_behind_the_scale_factor( - self, monkeypatch - ) -> None: - broken = json.loads(json.dumps(_CELLS)) - for tier in broken["tiers"]: + @staticmethod + def _scaled_cells(factor: float) -> dict: + scaled = json.loads(json.dumps(_CELLS)) + for tier in scaled["tiers"]: for measure in ("premium", "employee_contribution"): - for row in broken["private_state_2025"][tier][measure]["rows"].values(): + for row in scaled["private_state_2025"][tier][measure]["rows"].values(): for size in ("total", "lt50", "50plus"): if row[size] is not None: - row[size] /= 100.0 - for row in broken["public_division_2024"][tier][measure][ + row[size] *= factor + for row in scaled["public_division_2024"][tier][measure][ "rows" ].values(): for column in row: - row[column] /= 100.0 + row[column] *= factor + return scaled + + def test_a_unit_error_in_the_cells_cannot_hide_behind_the_scale_factor( + self, monkeypatch + ) -> None: + # Dollars read as hundreds of dollars. + broken = self._scaled_cells(0.01) monkeypatch.setattr(esi, "load_meps_ic_esi_premium_cells", lambda: broken) frame = _run(_realistic_rows()) - total = _weights(frame) @ _values(frame, EMPLOYER) + total = us_esi_premiums_summary(frame)["anchor_universe_employer_total"] assert total == pytest.approx(ANCHOR) # the scale factor hid the level gate = us_esi_premiums_signal_gate(frame) - assert any("the cell table is broken" in f for f in gate.failures) + assert any("units are broken" in f for f in gate.failures) - def test_an_export_frame_without_raw_columns_is_still_graded(self) -> None: + def test_a_level_error_inside_the_band_is_outside_this_gates_reach( + self, monkeypatch + ) -> None: + # The documented limit: every cell 20% low still averages inside the + # raw-mean band, the scale factor absorbs it, and this gate passes. A + # re-pinned table is reviewed against the PDFs, not caught here. + low = self._scaled_cells(0.8) + monkeypatch.setattr(esi, "load_meps_ic_esi_premium_cells", lambda: low) + gate = us_esi_premiums_signal_gate(_run(_realistic_rows())) + assert gate.passed, gate.failures + low_band, high_band = esi._RAW_MEAN_BAND + mean = gate.details["raw_employer_share_mean_per_positive_person"] + assert low_band <= mean <= high_band + + @pytest.mark.parametrize( + "column", + [c for c in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS if c != "state_fips"], + ) + def test_a_frame_without_a_raw_column_cannot_be_certified(self, column) -> None: frame = self._staged() - frame.table("person").drop( - columns=[ - column - for column in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS - if column != "state_fips" - ], - inplace=True, - ) + frame.table("person").drop(columns=[column], inplace=True) gate = us_esi_premiums_signal_gate(frame) - assert gate.passed, gate.failures + assert not gate.passed + assert gate.details["source_columns_missing"] == [column] + assert any("assignment provenance incomplete" in f for f in gate.failures) assert "scale_factor" not in gate.details - frame.table("person")[EMPLOYER] = 0.0 - assert not us_esi_premiums_signal_gate(frame).passed + + def test_a_misassignment_cannot_pass_by_dropping_a_raw_column(self) -> None: + # Reverse the premium vector: the weighted total is unchanged, the + # people who carry it are wrong. With the raw columns the gate names + # the misassignment; with one dropped it still fails, on provenance. + frame = _run(_realistic_rows()) # unit weights: the total is preserved + person = frame.table("person") + before = person[EMPLOYER].sum() + person[EMPLOYER] = person[EMPLOYER].to_numpy()[::-1].copy() + assert person[EMPLOYER].sum() == pytest.approx(before) + with_raw = us_esi_premiums_signal_gate(frame) + assert not with_raw.passed + assert any("outside employed policyholders" in f for f in with_raw.failures) + person.drop(columns=["NOEMP"], inplace=True) + without = us_esi_premiums_signal_gate(frame) + assert not without.passed + assert any("assignment provenance incomplete" in f for f in without.failures) + anchor = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not anchor.passed + assert any("assignment provenance incomplete" in f for f in anchor.failures) # --- the release anchor gate -------------------------------------------------------- class TestAnchorGate: - def _export(self, *, scale: float = 1.0, pre_tax_scale: float = 1.0) -> Frame: - """A calibrated export: engine inputs only, reweighted by ``scale``.""" + def _export(self, *, weight_scale: float = 1.0) -> Frame: + """A calibrated export: the staged people at rescaled weights.""" frame = _run(_realistic_rows(), weights=np.linspace(10, 900, 400)) - person = frame.table("person") - person.drop( - columns=list( - set(US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS) & set(person.columns) - ), - inplace=True, - ) - person[EMPLOYER] *= scale - person[PRE_TAX] *= pre_tax_scale - return frame + return _reweighted(frame, frame.weights_for("household").values * weight_scale) @pytest.mark.parametrize("scale", [0.951, 1.0, 1.049]) def test_passes_inside_the_stated_tolerance(self, scale) -> None: gate = us_esi_premiums_anchor_gate( - self._export(scale=scale), time_period=TIME_PERIOD + self._export(weight_scale=scale), time_period=TIME_PERIOD ) assert gate.passed, gate.failures assert gate.details["relative_error"] == pytest.approx(scale - 1.0) @@ -797,19 +933,55 @@ def test_passes_inside_the_stated_tolerance(self, scale) -> None: @pytest.mark.parametrize("scale", [0.94, 1.06]) def test_fails_outside_the_stated_tolerance(self, scale) -> None: gate = us_esi_premiums_anchor_gate( - self._export(scale=scale), time_period=TIME_PERIOD + self._export(weight_scale=scale), time_period=TIME_PERIOD ) assert not gate.passed - assert any("from BEA NIPA 7.8 line 17" in f for f in gate.failures) + assert any("from NHE Table 24" in f for f in gate.failures) - def test_the_retiree_inclusive_nhe_total_is_an_upper_bound(self) -> None: - # NHE is 7.2% above BEA, so the tolerance fails first; the bound must - # also be named for a release that drifted to the NHE concept. + def test_the_verdict_is_over_every_policyholder_not_the_column(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) + details = gate.details + assert details["anchor_universe_employer_total"] == pytest.approx(ANCHOR) + assert details["employer_premium_total"] < ANCHOR + assert details["employer_premium_total"] + details[ + "other_policyholder_employer_total" + ] == pytest.approx(ANCHOR) + assert details["employer_premium_total"] == pytest.approx( + ANCHOR * details["employed_share_of_anchor_universe"] + ) + + def test_reweighting_workers_against_other_policyholders_is_red(self) -> None: + # Calibration that keeps the column's total but piles weight onto + # retiree policyholders moves the anchor-universe total and the split. + frame = _run(_realistic_rows(), weights=np.full(400, 100.0)) + person = frame.table("person") + retiree = (person["PEMLR"].to_numpy() == 5) & ( + person["NOW_OWNGRP"].to_numpy() == 1 + ) + weights = np.where(retiree, 500.0, 100.0) gate = us_esi_premiums_anchor_gate( - self._export(scale=1.08), time_period=TIME_PERIOD + _reweighted(frame, weights), time_period=TIME_PERIOD + ) + assert gate.details["employer_premium_total"] == pytest.approx( + _column_total(frame) + ) + assert not gate.passed + assert any("from NHE Table 24" in f for f in gate.failures) + assert any("employed policyholders carry" in f for f in gate.failures) + + def test_the_cross_checks_are_recorded_not_gated(self) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) + assert gate.passed, gate.failures + # BEA's estimate of the same concept is 6.7% below NHE's for 2024. + assert gate.details["cross_check_ratio"] == pytest.approx(ANCHOR / BEA) + assert ANCHOR / BEA - 1 > esi.ANCHOR_RELATIVE_TOLERANCE + active = gate.details["private_active_cross_check"] + assert active["meps_ic_private_active_employer_total"] == pytest.approx( + 668.6e9, rel=5e-4 + ) + assert active["ratio"] == pytest.approx( + active["employer_premium_private_sector"] / 668.6e9, rel=5e-4 ) - assert any("exceeds the retiree-inclusive NHE" in f for f in gate.failures) - assert esi.ANCHOR_RELATIVE_TOLERANCE < 1_047.0e9 / ANCHOR - 1.0 @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) def test_an_absent_column_is_red(self, column) -> None: @@ -827,27 +999,63 @@ def test_a_zero_mass_column_is_red(self, column) -> None: assert not gate.passed assert f"{column}: zero weighted mass." in gate.failures - def test_pre_tax_premiums_cannot_exceed_nhe_employee_contributions(self) -> None: + @pytest.mark.parametrize("column", ["NOW_OWNGRP", "NOW_HIPAID", "PEMLR", "NOEMP"]) + def test_an_export_without_a_raw_column_is_red(self, column) -> None: + # The anchor counts every policyholder, so the gate cannot grade a + # frame that no longer says who they are. frame = self._export() - total = _weights(frame) @ _values(frame, PRE_TAX) - gate = us_esi_premiums_anchor_gate( - self._export(pre_tax_scale=1.01 * 382.1e9 / total), - time_period=TIME_PERIOD, - ) - assert any("employee contribution" in f for f in gate.failures) + frame.table("person").drop(columns=[column], inplace=True) + gate = us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD) + assert not gate.passed + assert any("assignment provenance incomplete" in f for f in gate.failures) + assert "anchor_universe_employer_total" not in gate.details - def test_a_year_without_an_anchor_is_red(self) -> None: - gate = us_esi_premiums_anchor_gate(self._export(), time_period=2031) + @pytest.mark.parametrize("year", [2025, 2031]) + def test_a_year_without_an_anchor_is_red(self, year) -> None: + gate = us_esi_premiums_anchor_gate(self._export(), time_period=year) assert not gate.passed assert any( - "no BEA NIPA 7.8 line 17 anchor for 2031" in f for f in gate.failures + f"no NHE Table 24 employer-contribution anchor for {year}" in f + for f in gate.failures ) def test_details_serialize_for_the_release_evidence_file(self) -> None: gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) payload = json.loads(json.dumps(dict(gate.details))) assert payload["anchor"]["sha256"] == esi.EMPLOYER_PREMIUM_ANCHOR["sha256"] - assert payload["cross_check"]["employer_contribution"]["2024"] == 1_047.0e9 + assert payload["anchor"]["values"]["2024"] == ANCHOR + assert payload["cross_check"]["values"]["2024"] == BEA + + +class TestOperatorBoundary: + def test_a_raw_source_frame_that_carries_the_outputs_is_refused(self) -> None: + from microcosm.build.us_runtime.operator_boundary import ( + PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES, + assert_operator_free_source_frame, + ) + + assert PRE_ASSEMBLY_OPERATOR_OUTPUT_FAMILIES["esi_premiums"] == { + "person": frozenset(US_ESI_PREMIUMS_OUTPUT_COLUMNS) + } + assert_operator_free_source_frame(_frame(_realistic_rows(40)), label="raw") + with pytest.raises(ValueError, match="esi_premiums:person="): + assert_operator_free_source_frame(_run(_realistic_rows(40)), label="raw") + + +class TestUnassignedRefusal: + def test_a_staged_frame_and_a_frame_without_the_columns_pass(self) -> None: + refuse_unassigned_us_esi_premiums(_run(_realistic_rows(60)), consumer="t") + refuse_unassigned_us_esi_premiums(_frame(_realistic_rows(60)), consumer="t") + + @pytest.mark.parametrize("column", US_ESI_PREMIUMS_OUTPUT_COLUMNS) + def test_a_null_on_any_row_is_refused(self, column) -> None: + # ACS-spine rows pooled beside an ASEC donor carry no assignment. + frame = _run(_realistic_rows(60)) + frame.table("person").loc[[3, 4], column] = np.nan + with pytest.raises(SourceRuntimeError, match="never filled") as error: + refuse_unassigned_us_esi_premiums(frame, consumer="ACS local release") + assert "ACS local release" in str(error.value) + assert f"'{column}': 2" in str(error.value) # --- properties and the differential reference ------------------------------------ @@ -871,8 +1079,6 @@ def _person_rows(draw) -> dict: "PEMLR": draw(st.sampled_from([0, 1, 1, 1, 2, 3, 4, 5, 6, 7])), "NOEMP": draw(st.integers(0, 6)), "PEIO1COW": draw(st.sampled_from([0, 1, 2, 3, 4, 4, 4, 5, 6, 7, 8])), - "PHIP_VAL": draw(st.sampled_from([0, 0, 350, 1_800, 7_400, 22_000])), - "WSAL_VAL": draw(st.sampled_from([0, 12_000, 48_000, 250_000])), "state_fips": draw(st.sampled_from(_STATES)), } @@ -898,18 +1104,17 @@ def _populations(draw): return rows, weights -def _reference(rows: list[dict], weights: list[float], *, seed: int): +def _reference(rows: list[dict], weights: list[float]) -> list[float]: """A scalar restatement of the declared rules, one person at a time.""" tiers = {1: "family", 2: "employee_plus_one", 3: "single"} raw: list[float] = [] - pre_tax: list[float] = [] - for index, partial in enumerate(rows): + in_column: list[bool] = [] + for partial in rows: row = {**_BASE_ROW, **partial} + holder = row["NOW_OWNGRP"] == 1 in_universe = ( - row["NOW_OWNGRP"] == 1 - and row["PEMLR"] in (1, 2) - and row["PEIO1COW"] in (1, 2, 3, 4, 5, 6) + holder and row["PEMLR"] in (1, 2) and row["PEIO1COW"] in (1, 2, 3, 4, 5, 6) ) size = ( "total" @@ -919,21 +1124,16 @@ def _reference(rows: list[dict], weights: list[float], *, seed: int): else "50plus" ) share = 0.0 - if in_universe and row["NOW_HIPAID"] in (1, 2): + if holder and row["NOW_HIPAID"] in (1, 2): tier = tiers[row["NOW_GRPFTYP2"]] state = f"{row['state_fips']:02d}" + government = row["PEIO1COW"] in (1, 2, 3) + # No employer class: the State's all-sizes private cell. + cell_size = size if row["PEIO1COW"] in (4, 5, 6) else "total" cell = {} for measure in ("premium", "employee_contribution"): private = _CELLS["private_state_2025"][tier][measure]["rows"] - if row["PEIO1COW"] in (4, 5, 6): - value = private[state][size] - if value is None: - value = ( - private["US"][size] - * private[state]["total"] - / private["US"]["total"] - ) - else: + if government: division = _CELLS["state_census_division"][state] column = "state" if row["PEIO1COW"] == 2 else "all_state_and_local" value = ( @@ -943,32 +1143,37 @@ def _reference(rows: list[dict], weights: list[float], *, seed: int): * private["US"]["total"] / _CELLS["private_national_2024"][tier][measure]["total"] ) + else: + value = private[state][cell_size] + if value is None: + value = ( + private["US"][cell_size] + * private[state]["total"] + / private["US"]["total"] + ) cell[measure] = value + unpaid = ( + _CELLS["no_contribution_share_2025"][tier][ + "50plus" if government else cell_size + ] + / 100.0 + ) share = ( cell["premium"] if row["NOW_HIPAID"] == 1 - else max(cell["premium"] - cell["employee_contribution"], 0.0) + else max( + cell["premium"] - cell["employee_contribution"] / (1.0 - unpaid), + 0.0, + ) ) raw.append(share) - eligible = ( - in_universe - and row["NOW_HIPAID"] in (2, 3) - and row["WSAL_VAL"] > 0 - and row["PHIP_VAL"] > 0 - ) - draw = stable_identity_uniforms( - [f"2024:{index + 1}:1"], seed=seed, salt=PRE_TAX - )[0] - offer = _CELLS["pretax_contribution_2025"]["rows"][size] - probability = ( - offer["pretax_contribution_offer_percent"] - / offer["health_insurance_offer_percent"] - ) - pre_tax.append( - float(row["PHIP_VAL"]) if eligible and draw < probability else 0.0 - ) + in_column.append(in_universe) + # One factor over every policyholder; only the column's people keep it. total = sum(weight * share for weight, share in zip(weights, raw, strict=True)) - return [share * ANCHOR / total for share in raw], pre_tax + return [ + share * ANCHOR / total if keep else 0.0 + for share, keep in zip(raw, in_column, strict=True) + ] _PROPERTY_SETTINGS = settings( @@ -983,9 +1188,9 @@ def _reference(rows: list[dict], weights: list[float], *, seed: int): def test_stage_agrees_with_the_scalar_reference(population, seed) -> None: rows, weights = population result = _run(rows, weights=weights, seed=seed) - employer, pre_tax = _reference(rows, weights, seed=seed) - np.testing.assert_allclose(_values(result, EMPLOYER), employer, rtol=1e-11) - assert _values(result, PRE_TAX).tolist() == pre_tax + np.testing.assert_allclose( + _values(result, EMPLOYER), _reference(rows, weights), rtol=1e-11 + ) @_PROPERTY_SETTINGS @@ -994,10 +1199,15 @@ def test_conservation_bounds_and_structural_zeros(population, seed) -> None: rows, weights = population result = _run(rows, weights=weights, seed=seed) person = result.table("person") - employer, pre_tax = _values(result, EMPLOYER), _values(result, PRE_TAX) - assert _weights(result) @ employer == pytest.approx(ANCHOR, rel=1e-11) - assert np.isfinite(employer).all() and np.isfinite(pre_tax).all() - assert (employer >= 0).all() and (pre_tax >= 0).all() + employer = _values(result, EMPLOYER) + summary = us_esi_premiums_summary(result) + assert summary["anchor_universe_employer_total"] == pytest.approx(ANCHOR, rel=1e-11) + assert _weights(result) @ employer == pytest.approx( + ANCHOR * summary["employed_share_of_anchor_universe"], rel=1e-11 + ) + assert _weights(result) @ employer <= ANCHOR * (1 + 1e-11) + assert np.isfinite(employer).all() + assert (employer >= 0).all() universe = ( person["NOW_OWNGRP"].eq(1) & person["PEMLR"].isin([1, 2]) @@ -1006,16 +1216,6 @@ def test_conservation_bounds_and_structural_zeros(population, seed) -> None: assert (employer[~universe] == 0).all() assert (employer[universe & person["NOW_HIPAID"].eq(3).to_numpy()] == 0).all() assert (employer[universe & person["NOW_HIPAID"].isin([1, 2]).to_numpy()] > 0).all() - phip = person["PHIP_VAL"].to_numpy(dtype=float) - assert ((pre_tax == 0) | (pre_tax == phip)).all() - eligible = ( - universe - & person["NOW_HIPAID"].isin([2, 3]).to_numpy() - & (person["WSAL_VAL"].to_numpy() > 0) - & (phip > 0) - ) - assert (pre_tax[~eligible] == 0).all() - summary = us_esi_premiums_summary(result) for key in ( "nonfinite_rows", "negative_rows", @@ -1023,8 +1223,6 @@ def test_conservation_bounds_and_structural_zeros(population, seed) -> None: "employer_premium_outside_universe_rows", "employer_premium_where_employer_pays_none_rows", "employer_premium_without_raw_share_rows", - "pre_tax_ineligible_rows", - "pre_tax_not_reported_premium_rows", ): assert summary[key] == 0, key assert summary["scale_factor_spread"] <= 1e-9 * summary["scale_factor"] @@ -1051,7 +1249,6 @@ def test_row_order_does_not_change_anyone(population, random) -> None: np.testing.assert_allclose( shuffled[EMPLOYER].to_numpy(), straight[EMPLOYER].to_numpy(), rtol=1e-11 ) - assert shuffled[PRE_TAX].tolist() == straight[PRE_TAX].tolist() @_PROPERTY_SETTINGS @@ -1063,37 +1260,106 @@ def test_weights_are_homogeneous_of_degree_minus_one(population, factor) -> None np.testing.assert_allclose( _values(scaled, EMPLOYER) * factor, _values(base, EMPLOYER), rtol=1e-10 ) - assert _weights(scaled) @ _values(scaled, EMPLOYER) == pytest.approx( - ANCHOR, rel=1e-11 - ) - assert _values(scaled, PRE_TAX).tolist() == _values(base, PRE_TAX).tolist() + assert _column_total(scaled) == pytest.approx(_column_total(base), rel=1e-10) + assert us_esi_premiums_summary(scaled)[ + "anchor_universe_employer_total" + ] == pytest.approx(ANCHOR, rel=1e-11) @_PROPERTY_SETTINGS -@given(_populations()) -def test_staging_is_idempotent_and_clone_safe(population) -> None: +@given(_populations(), st.integers(0, 2**31 - 1)) +def test_staging_is_deterministic_restageable_and_clone_safe(population, seed) -> None: rows, weights = population - once = _run(rows, weights=weights) - assert ( - us_esi_premiums_summary(_cloned(once))["clone_disagreement_source_persons"] == 0 + once = _run(rows, weights=weights, seed=seed) + # The assignment draws nothing, so another seed changes no one. + again = _run(rows, weights=weights, seed=seed + 1) + # Staging a staged frame again from its raw columns changes no one: the + # stage reads only the raw ASEC columns, never its own outputs. + stripped = _reweighted(once, once.weights_for("household").values) + stripped.table("person").drop( + columns=list(US_ESI_PREMIUMS_OUTPUT_COLUMNS), inplace=True ) + restaged = with_us_esi_premium_inputs(stripped, seed=seed, time_period=TIME_PERIOD) + for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: + assert _values(again, column).tolist() == _values(once, column).tolist() + assert _values(restaged, column).tolist() == _values(once, column).tolist() cloned = _cloned(once) + assert us_esi_premiums_summary(cloned)["clone_disagreement_source_persons"] == 0 for column in US_ESI_PREMIUMS_OUTPUT_COLUMNS: assert _weights(cloned) @ _values(cloned, column) == pytest.approx( _weights(once) @ _values(once, column), rel=1e-11, abs=1e-6 ) + assert us_esi_premiums_summary(cloned)[ + "anchor_universe_employer_total" + ] == pytest.approx(ANCHOR, rel=1e-11) + + +@_PROPERTY_SETTINGS +@given(_populations()) +def test_pricing_other_policyholders_never_raises_a_workers_premium(population) -> None: + rows, weights = population + priced = _run(rows, weights=weights) + person = priced.table("person") + universe = ( + person["NOW_OWNGRP"].eq(1) + & person["PEMLR"].isin([1, 2]) + & person["PEIO1COW"].isin([1, 2, 3, 4, 5, 6]) + ).to_numpy() + # The same frame with every policyholder outside the column uncovered: + # nothing is left to price but the column, so it carries the whole anchor. + workers_only = _run( + [ + row if keep else {**row, **_NO_COVERAGE} + for row, keep in zip(rows, universe, strict=True) + ], + weights=weights, + ) + assert _column_total(workers_only) == pytest.approx(ANCHOR, rel=1e-11) + assert ( + _values(priced, EMPLOYER) <= _values(workers_only, EMPLOYER) * (1 + 1e-11) + ).all() + assert _column_total(priced) <= ANCHOR * (1 + 1e-11) + + +@_PROPERTY_SETTINGS +@given( + st.sampled_from([1, 2, 3]), + st.sampled_from([1, 2, 3, 4, 5, 6]), + st.sampled_from([4, 5, 6]), + st.sampled_from(_STATES), + st.floats(0.5, 500.0, allow_nan=False), +) +def test_the_published_mix_reconciles_every_private_cell( + tier_code, noemp, sector, state, mass +) -> None: + tier = esi._TIER_BY_CODE[tier_code] + share = _no_contribution(tier, esi._SIZE_BY_NOEMP[noemp]) + cell = { + "NOW_GRPFTYP2": tier_code, + "NOW_GRPFTYP": 1 if tier_code in (1, 2) else 2, + "NOEMP": noemp, + "PEIO1COW": sector, + "state_fips": state, + } + result = _run( + [{**cell, "NOW_HIPAID": 1}, {**cell, "NOW_HIPAID": 2}], + weights=[mass * share, mass * (1 - share)], + ) + summary = us_esi_premiums_summary(result) + assert summary["raw_over_published_employer_mean"] == pytest.approx(1.0) # --- the registers the stage moves ------------------------------------------------- class TestRegisters: - def test_both_inputs_are_required_release_columns_with_no_exclusion(self) -> None: + def test_the_input_is_a_required_release_column_with_no_exclusion(self) -> None: required = us_release_input_coverage_required_columns() - assert {EMPLOYER, PRE_TAX} <= required - assert not {EMPLOYER, PRE_TAX} & set( - us_release_input_coverage_reviewed_exclusions() - ) + assert EMPLOYER in required + assert EMPLOYER not in set(us_release_input_coverage_reviewed_exclusions()) + # Not produced, so not required: the pre-tax employee premium waits + # for the engine's disjoint premium contract (policyengine-us#10046). + assert "pre_tax_health_insurance_premiums" not in required def test_the_parity_gap_register_no_longer_exempts_the_employer_premium( self, @@ -1105,7 +1371,7 @@ def test_the_parity_gap_register_no_longer_exempts_the_employer_premium( ) assert EMPLOYER not in payload["known_gaps"] - def test_shipped_reform_probes_bind_through_both_inputs(self) -> None: + def test_a_shipped_reform_probe_binds_through_the_input(self) -> None: probes = {probe.id: probe for probe in us_release_reform_coverage_probes()} employer = probes["employer_sponsored_insurance_premium_neutralization"] assert employer.neutralized_variable == EMPLOYER @@ -1113,12 +1379,9 @@ def test_shipped_reform_probes_bind_through_both_inputs(self) -> None: assert employer.effect_direction == "baseline_minus_reform" assert employer.expected_sign == "positive" assert 0 < employer.min_abs_effect < ANCHOR - pre_tax = probes["pre_tax_health_insurance_premium_neutralization"] - assert pre_tax.neutralized_variable == PRE_TAX - assert pre_tax.budget_measure == "income_tax" - assert pre_tax.expected_sign == "negative" + assert "pre_tax_health_insurance_premium_neutralization" not in probes - def test_the_nhe_target_family_is_fenced_as_a_broader_concept(self) -> None: + def test_the_nhe_target_family_is_gated_not_compiled(self) -> None: manifest = json.loads( files("microcosm.build.us") .joinpath("target_parity_manifest.json") @@ -1132,23 +1395,25 @@ def test_the_nhe_target_family_is_fenced_as_a_broader_concept(self) -> None: entry = families[family] assert entry["status"] == "reviewed_exclusion" assert entry["classification"] == "deferred" - assert "retiree" in entry["reason"] + assert "every current ESI policyholder" in entry["reason"] + assert "retirees" in entry["reason"] assert "us_esi_premiums_anchor_gate" in entry["reason"] absent = families["bea_nipa.private_group_health_insurance"] assert absent["classification"] == "source_absent" assert "B4923C" in absent["reason"] + assert "active and retired" in absent["fence"]["purpose"] @pytest.mark.parametrize( - ("pool", "scale", "holders", "mean", "pre_tax", "nhe_ratio"), + ("pool", "scale", "holders", "column", "mean", "share"), [ # The default pool (docs/us-asec-source-pins.md) and the historical # Build J/N/P pool; figures quoted in the docs and the registers. - ("2023_2025", 1.0703, 73.97e6, 13_209, 181.1e9, 0.985), - ("2022_2024", 1.0664, 74.06e6, 13_193, 174.6e9, 0.990), + ("2023_2025", 1.0404, 73.97e6, 925.7e9, 12_515, 0.8842), + ("2022_2024", 1.0345, 74.06e6, 923.9e9, 12_476, 0.8824), ], ) def test_the_receipts_reproduce_the_documented_measurements( - self, pool, scale, holders, mean, pre_tax, nhe_ratio + self, pool, scale, holders, column, mean, share ) -> None: receipt = json.loads( ( @@ -1163,21 +1428,41 @@ def test_the_receipts_reproduce_the_documented_measurements( assert receipt["anchor_gate"]["passed"] is True clone = receipt["puf_support_clone"] assert clone["signal_gate"]["passed"] and clone["anchor_gate"]["passed"] - assert clone["anchor_gate"]["employer_premium_total"] == pytest.approx( - ANCHOR, rel=1e-12 - ) - assert summary["employer_premium_total"] == pytest.approx(ANCHOR, rel=1e-12) + for totals in (summary, clone["anchor_gate"]): + assert totals["anchor_universe_employer_total"] == pytest.approx( + ANCHOR, rel=1e-12 + ) + assert totals["employer_premium_total"] == pytest.approx(column, rel=1e-4) assert summary["cells_sha256"] == esi._CELLS_SHA256 + assert summary["source_columns_missing"] == [] assert summary["scale_factor"] == pytest.approx(scale, abs=5e-5) + assert summary["employed_share_of_anchor_universe"] == pytest.approx( + share, abs=5e-5 + ) assert summary["employer_premium_positive_persons"] == pytest.approx( holders, rel=1e-3 ) assert summary["employer_premium_mean_per_positive_person"] == pytest.approx( mean, abs=1 ) - assert summary["pre_tax_premium_total"] == pytest.approx(pre_tax, rel=1e-3) - assert receipt["nhe_concept_check"]["raw_all_policyholders_over_nhe"] == ( - pytest.approx(nhe_ratio, abs=5e-4) + details = receipt["anchor_gate"]["details"] + assert details["cross_check_ratio"] == pytest.approx(ANCHOR / BEA) + active = details["private_active_cross_check"] + assert active["meps_ic_private_active_employer_total"] == pytest.approx( + 668.6e9, rel=5e-4 + ) + assert 1.06 < active["ratio"] < 1.07 + sensitivity = receipt["other_policyholder_sensitivity"] + assert sensitivity["as_built_other_priced_as_active"][ + "employer_premium_total" + ] == pytest.approx(summary["employer_premium_total"]) + assert sensitivity["other_not_priced_all_anchor_on_employed"][ + "employer_premium_total" + ] == pytest.approx(ANCHOR) + assert ( + summary["employer_premium_total"] + < sensitivity["other_65_plus_priced_at_half"]["employer_premium_total"] + < ANCHOR ) restored = ["NOW_OWNGRP", "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2"] for source in receipt["pool_census_person_columns"]: @@ -1219,6 +1504,20 @@ def test_row_parser_reads_suppression_and_reliability_marks(self) -> None: virginia, _ = tool._row_values(block, "Virginia", 8, "II.D.2") assert virginia[0] == 7000 + def test_a_flagged_or_suppressed_national_row_is_refused(self) -> None: + tool = _load_cells_tool() + pin = next(pin for pin in tool.PDFS if pin.key == "private_state_2025") + head = "Table II.C.4.a Percent of enrollees: United States, 2025" + good = "United States 13.0% 52.7% 32.6% 24.4% 13.4% 4.8% 35.5% 8.7%" + row = tool._national_row([head, good], pin, "II.C.4.a") + assert (row["total"], row["lt50"], row["50plus"]) == (13.0, 35.5, 8.7) + # A flag on a column the stage does not read is tolerated. + tolerated = good.replace("4.8%", "4.8% *") + assert tool._national_row([head, tolerated], pin, "II.C.4.a")["lt50"] == 35.5 + for bad in (good.replace("35.5%", "35.5% *"), good.replace("8.7%", "--")): + with pytest.raises(tool.ExtractionError, match="suppressed or flagged"): + tool._national_row([head, bad], pin, "II.C.4.a") + def test_row_parser_refuses_a_missing_duplicated_or_short_row(self) -> None: tool = _load_cells_tool() row = "Ohio 1 2 3" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 919d926661..657d5d7936 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -6411,17 +6411,20 @@ def fake_smoke(*, simulate, period): } # microcosm#454: the ESI premium signal gate graded the base before the # solve and the export after it; the anchor gate graded the export at the - # release period and its verdict is the bound esi_premiums evidence. + # release period and its verdict is the bound esi_premiums evidence, with + # both signal verdicts recorded beside it. assert captured["esi_premiums_signal_gate_calls"] == 2 assert captured["esi_premiums_anchor_periods"] == [builder.PERIOD] + signal_evidence = {"evaluated": True, "passed": True, "failures": []} assert json.loads((release_dir / "esi_premiums.json").read_text()) == { - "schema_version": 1, + "schema_version": 2, "enforced": True, "esi_premiums": { "passed": True, "failures": [], "details": {"checked": True}, }, + "signal": {"base_frame": signal_evidence, "export_frame": signal_evidence}, } for block in (build_manifest, release_manifest["build"]): assert block["gate_evidence"] == expected_gate_evidence @@ -16964,3 +16967,25 @@ def test_main_reports_actual_dry_run_outcome( if expected_status == "failed": assert str(exit_code) in run_events[0]["message"] assert run_events[0]["details"]["failure_class"] == "build_failure" + + +def test_esi_premium_signal_evidence_keeps_a_waived_failure_on_the_record() -> None: + """microcosm#454: --allow-esi-premium-gaps records, it does not erase. + + The evidence file carries the base and export signal verdicts, so a build + that waived an assignment failure still says which check was red. A gate + that never ran (it crashed under earlier failures) is recorded as such. + """ + + builder = _load_builder_module() + failed = builder.GateResult( + name="esi_premiums_signal", + passed=False, + failures=("ESI premiums: 100 employer premium(s) outside employed ...",), + ) + assert builder._esi_premiums_signal_evidence(failed) == { + "evaluated": True, + "passed": False, + "failures": ["ESI premiums: 100 employer premium(s) outside employed ..."], + } + assert builder._esi_premiums_signal_evidence(None) == {"evaluated": False} diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py index 896f3cf3b9..789b149a05 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spec_bundle.py @@ -258,7 +258,7 @@ def test_legacy_seed_vintage_and_publication_grammars_are_pinned( "identity_generation": 1, "seed_protocol": LEGACY_V1_PROTOCOL.id, } - assert len(LEGACY_V1_PROTOCOL.sites) == 54 + assert len(LEGACY_V1_PROTOCOL.sites) == 53 assert len(LEGACY_V1_PROTOCOL.streams) == 14 assert LEGACY_V1_PROTOCOL.site("survey_sample_asec").default == 578 assert LEGACY_V1_PROTOCOL.site("puf_live_aggregate_disaggregation").default == 0 diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index 777cf5b5eb..da135839f6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -266,8 +266,8 @@ "education_assistance_source.py", "eligibility_inputs.py", "engine_lifecycle.py", - # MEPS-IC employer-premium and pre-tax premium inputs (#454): a - # base-builder pre-clone stage keyed on restored ASEC coverage fields, + # MEPS-IC employer-premium input (#454): a base-builder pre-clone + # stage keyed on restored ASEC coverage fields, # not yet a multispine pool operator. Reads no spine or support-channel # column and remains subject to the all-runtime source-identity scan. "esi_premiums.py", diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 3c150fc2d6..416be017a9 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "e9360b25997322886028b7a8a8964007201e2b36b7bbd341d0b483093e026950", + "spec_sha256": "a1e98deaa1f5a451c7d0e1f380a7bf50a82b0cbc94753b885b7cf9a11e7d0005", }, } diff --git a/test_support/microcosm_build/spec_engine_inventory_coverage.py b/test_support/microcosm_build/spec_engine_inventory_coverage.py index 26072ef5e6..1d051f5c4d 100644 --- a/test_support/microcosm_build/spec_engine_inventory_coverage.py +++ b/test_support/microcosm_build/spec_engine_inventory_coverage.py @@ -124,9 +124,9 @@ def binding() -> dict[str, object]: "producer_virtual_resources": 75, "release_rungs": 5, "resolved_references": 334, - "seed_owner_bindings": 113, - "seed_owner_rows": 55, - "seed_sites": 54, + "seed_owner_bindings": 112, + "seed_owner_rows": 54, + "seed_sites": 53, "seed_streams": 14, "source_operators": 16, "source_stages": 38, diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index b03c7c8050..acfe2ce7ef 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91" + "e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 464a658fd9..a9f430dfe5 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -80,6 +80,7 @@ from microcosm.build.us_runtime import target_loss_weights as loss_weighting from microcosm.build.us_runtime.acs_local_hours import acs_local_hours_signal_gate +from microcosm.build.us_runtime.esi_premiums import refuse_unassigned_us_esi_premiums _TOOLS_DIR = Path(__file__).resolve().parent _REPO_ROOT = _TOOLS_DIR.parent @@ -502,9 +503,17 @@ def fill_reviewed_nulls( Every registered (entity, column) has its NaN filled with the pe-us variable's own default; NaN in any engine-input column NOT in the register is a hard error with a per-spine diagnostic — an artifact - defect, surfaced not filled. + defect, surfaced not filled. The two ESI premium inputs are never + filled, registered or not: a null there refuses the build. """ + # microcosm#454: the ESI premium inputs exist only where the + # meps_esi_premiums stage ran (the ASEC donor rows). Refuse before any + # fill: a registered null here would become a silent zero on the ACS spine. + refuse_unassigned_us_esi_premiums( + frame, consumer="ACS local release (fill_reviewed_nulls)" + ) + from policyengine_us import CountryTaxBenefitSystem from microcosm.build.us_runtime.base_pool import spine_column diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index f8b7b21f53..3214e4da4e 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -1308,11 +1308,12 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: action="store_true", help=( "Diagnostic escape hatch (microcosm#454): record the ESI premium " - "gates — employer_sponsored_insurance_premiums and " - "pre_tax_health_insurance_premiums absent, zero-mass, or off the " - "BEA NIPA 7.8 anchor — without failing the build (for example on " - "a base built before the meps_esi_premiums stage). Release builds " - "must leave this unset." + "gates — employer_sponsored_insurance_premiums absent, zero-mass, " + "missing its raw ASEC coverage columns, or off the CMS NHE Table 24 " + "anchor — without failing the build (for example on a base built " + "before the meps_esi_premiums stage). The failures are still " + "written to esi_premiums.json. Release builds must leave this " + "unset." ), ) parser.add_argument( @@ -7444,6 +7445,22 @@ def _engine_input_variables() -> tuple[str, ...]: return tuple(PolicyEngineUSEngine().variables()) +def _esi_premiums_signal_evidence(gate: GateResult | None) -> dict[str, object]: + """One ESI premium signal verdict for the release evidence file. + + Failures are kept verbatim so a build that waived them + (``--allow-esi-premium-gaps``) still records what was red. + """ + + if gate is None: + return {"evaluated": False} + return { + "evaluated": True, + "passed": gate.passed, + "failures": list(gate.failures), + } + + def _input_mass_reference_gate( base_frame: Frame, *, @@ -14133,8 +14150,8 @@ def _main(argv: Sequence[str] | None = None) -> int | None: for failure in other_health_insurance_gate.failures ) ) - # microcosm#454: the meps_esi_premiums base stage owns both ESI premium - # inputs; refuse a base that lacks or flattens them before the solve. + # microcosm#454: the meps_esi_premiums base stage owns the ESI employer + # premium input; refuse a base that lacks or flattens it before the solve. esi_premiums_base_gate = us_esi_premiums_signal_gate(base_frame) if not esi_premiums_base_gate.passed and not args.allow_esi_premium_gaps: if telemetry is not None: @@ -15350,11 +15367,13 @@ def _main(argv: Sequence[str] | None = None) -> int | None: failures=list(input_coverage_gate.failures), force_upload=True, ) - # microcosm#454: the calibrated ESI premium totals against their external - # anchors (BEA NIPA 7.8 line 17 within tolerance; at most the - # retiree-inclusive NHE Table 24), red when either column is absent or - # zero-mass. Runs on the calibrated export for BOTH the dense and sparse - # default paths; the verdict ships as the esi_premiums gate evidence. + # microcosm#454: the calibrated ESI premium total over the anchor's + # universe (every policyholder) against CMS NHE Table 24 within tolerance, + # red when either column is absent or zero-mass or the raw ASEC coverage + # columns are gone. Runs on the calibrated export for BOTH the dense and + # sparse default paths; the verdict ships as the esi_premiums gate + # evidence, with the base and export signal verdicts beside it so a waived + # assignment failure stays on the record. try: esi_premiums_anchor_gate = us_esi_premiums_anchor_gate( export_frame, time_period=PERIOD @@ -15380,13 +15399,21 @@ def _main(argv: Sequence[str] | None = None) -> int | None: esi_premiums_path.write_text( json.dumps( { - "schema_version": 1, + "schema_version": 2, "enforced": not args.allow_esi_premium_gaps, "esi_premiums": { "passed": esi_premiums_anchor_gate.passed, "failures": list(esi_premiums_anchor_gate.failures), "details": dict(esi_premiums_anchor_gate.details), }, + "signal": { + "base_frame": _esi_premiums_signal_evidence( + esi_premiums_base_gate + ), + "export_frame": _esi_premiums_signal_evidence( + esi_premiums_export_gate + ), + }, }, indent=2, sort_keys=True, diff --git a/tools/build_us_meps_ic_esi_cells.py b/tools/build_us_meps_ic_esi_cells.py index 18a6819659..ef1126c93c 100644 --- a/tools/build_us_meps_ic_esi_cells.py +++ b/tools/build_us_meps_ic_esi_cells.py @@ -1,9 +1,9 @@ """Regenerate the packaged MEPS-IC employer-premium cell table (microcosm#454). -The ``meps_esi_premiums`` source stage assigns each employed ESI policyholder -the MEPS-IC employer share of the premium for their coverage tier, employer -size and state (or census division for State and local government -employers). Those cells are published only as PDF tables, so this tool pins +The ``meps_esi_premiums`` source stage assigns each ESI policyholder the +MEPS-IC employer share of the premium for their coverage tier, employer size +and state (or census division for State and local government employers). +Those cells are published only as PDF tables, so this tool pins the official AHRQ PDFs by URL, byte length and SHA-256, extracts the cells the stage reads with ``pdftotext -layout`` (poppler), and writes them, with their provenance, to @@ -12,11 +12,17 @@ Every value is copied as published. Suppressed cells (``--``) stay ``null``: the fallback for them is stage logic, declared in the stage manifest, not -data. Each table's United States row is checked against the AHRQ national -figure it must reproduce, and a parse that does not find exactly one row per -state, division or firm-size label refuses. +data. The premium tables' United States rows are checked against the AHRQ +national figures they must reproduce, and a parse that does not find exactly +one row per state, division or firm-size label refuses. -Run (downloads the four PDFs, about 34 MB, into ``--pdf-dir``):: +Two national blocks ride along: the share of enrollees whose coverage +required no employee contribution (Tables II.C/D/E.4.a), which turns the +average employee contribution into the contribution of those who pay one; and +the private-sector enrollment rows (Tables II.B.1, II.B.2, II.B.2.b and +II.C/D/E.4) behind the stage's active-employee cross-check. + +Run (downloads the three PDFs, about 25 MB, into ``--pdf-dir``):: uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic @@ -73,8 +79,10 @@ def filename(self) -> str: survey_year=2025, description=( "MEPS-IC 2025 Series II (private sector, by firm size and State): " - "Tables II.C/D/E.1 (average total premium per enrolled employee) " - "and II.C/D/E.2 (average employee contribution)" + "Tables II.C/D/E.1 (average total premium per enrolled employee), " + "II.C/D/E.2 (average employee contribution), II.C/D/E.4.a (percent " + "of enrollees whose coverage required no employee contribution), " + "and the enrollment rows II.B.1, II.B.2, II.B.2.b and II.C/D/E.4" ), ), PinnedPdf( @@ -86,7 +94,7 @@ def filename(self) -> str: description=( "MEPS-IC 2024 Series II (private sector, by firm size and State); " "the United States rows age the 2024 State and local government " - "cells to 2025" + "cells to 2025 and carry the 2024 private-sector enrollment" ), ), PinnedPdf( @@ -102,19 +110,6 @@ def filename(self) -> str: "for this series (the 2025 series is not yet released)." ), ), - PinnedPdf( - key="private_national_2025", - url=_MEPS_ROOT + "national/series_1/2025/ic25_ia_g.pdf", - size_bytes=9_210_366, - sha256="cceba5da68e41caf1b03b83f93d1c20fc1aa1b3ccd0eefc4a9dfc80754c950f3", - survey_year=2025, - description=( - "MEPS-IC 2025 Series I (private sector, national): Table I.A.2 " - "(percent of establishments that offer health insurance) and " - "Table I.A.2.j (percent of establishments that offer pretax " - "employee contributions to health insurance), by firm size" - ), - ), ) TIERS = ("single", "employee_plus_one", "family") @@ -276,7 +271,7 @@ def pdftotext_version() -> str: def _table_block(lines: list[str], table_id: str) -> tuple[int, list[str]]: - head = re.compile(rf"^Table {re.escape(table_id)} (Average|Percent)") + head = re.compile(rf"^Table {re.escape(table_id)} (Average|Percent|Number)") starts = [ index for index, line in enumerate(lines) @@ -391,31 +386,65 @@ def _series_iii_table(lines, pin: PinnedPdf, table_id: str) -> dict: return {"text_line": start + 1, "title": block[0].strip(), "rows": rows} -def _pretax_offer_shares(lines, pin: PinnedPdf) -> dict: - """Firm-size pretax-contribution and health-insurance offer rates.""" - - _start, offer = _table_block(lines, "I.A.2") - offer_values, _ = _row_values(offer, "United States", 8, "I.A.2") - offer_by_column = dict(zip(_SERIES_II_COLUMNS, offer_values, strict=True)) - start_j, pretax = _table_block(lines, "I.A.2.j") - out = {} - for key, label in ( - ("total", "United States"), - ("lt50", "Less than 50 employees"), - ("50plus", "50+ employees"), - ): - values, unreliable = _row_values(pretax, label, 3, "I.A.2.j") - if unreliable[0]: - raise ExtractionError(f"Table I.A.2.j {label} is flagged unreliable.") - out[key] = { - "pretax_contribution_offer_percent": values[0], - "health_insurance_offer_percent": offer_by_column[key], - } +def _full_title(block: list[str]) -> str: + """A table title through its ``United States, `` line (it may wrap).""" + + title = "" + for line in block[:4]: + title = f"{title} {line.strip()}".strip() + if re.search(r"United States, \d{4}$", title): + return title + raise ExtractionError(f"Table title does not end in a survey year: {block[0]!r}.") + + +def _national_row(lines, pin: PinnedPdf, table_id: str) -> dict: + """The United States row of one Series II table, kept firm-size columns.""" + + start, block = _table_block(lines, table_id) + if _table_year(block, table_id) != pin.survey_year: + raise ExtractionError(f"Table {table_id} is not survey year {pin.survey_year}.") + values, unreliable = _row_values( + block, "United States", len(_SERIES_II_COLUMNS), table_id + ) + by_column = dict(zip(_SERIES_II_COLUMNS, values, strict=True)) + flags = dict(zip(_SERIES_II_COLUMNS, unreliable, strict=True)) + kept = {column: by_column[column] for column in _KEPT_SERIES_II_COLUMNS} + bad = [ + column + for column in _KEPT_SERIES_II_COLUMNS + if kept[column] is None or flags[column] + ] + if bad: + raise ExtractionError( + f"Table {table_id}: United States {bad} suppressed or flagged unreliable." + ) + return {"text_line": start + 1, "title": _full_title(block), **kept} + + +def _no_contribution_shares(lines, pin: PinnedPdf) -> dict: + """Percent of enrollees, by tier, whose coverage required no contribution. + + National by firm size only: most State cells of these tables are flagged + unreliable or suppressed. + """ + return { - "text_line": start_j + 1, - "title": pretax[0].strip(), - "offer_title": offer[0].strip(), - "rows": out, + tier: _national_row(lines, pin, f"II.{_TIER_LETTER[tier]}.4.a") + for tier in TIERS + } + + +def _private_enrollment(lines, pin: PinnedPdf) -> dict: + """National private-sector enrollment rows for the active-employee check.""" + + return { + "employees": _national_row(lines, pin, "II.B.1"), + "offer_percent": _national_row(lines, pin, "II.B.2"), + "enrolled_percent": _national_row(lines, pin, "II.B.2.b"), + "tier_share_percent": { + tier: _national_row(lines, pin, f"II.{_TIER_LETTER[tier]}.4") + for tier in TIERS + }, } @@ -467,7 +496,7 @@ def build_payload(pdf_dir: Path) -> dict: _start, division_block = _table_block(texts["private_state_2025"], "II.C.1") divisions = _state_divisions(division_block) return { - "schema_version": 1, + "schema_version": 2, "issue": "PolicyEngine/microcosm#454", "description": ( "MEPS-IC average total premium and average employee contribution " @@ -475,10 +504,12 @@ def build_payload(pdf_dir: Path) -> dict: "meps_esi_premiums source stage: private-sector cells by State and " "firm size (2025), State and local government cells by census " "division (2024, the latest published year), the 2024 private " - "national rows that age the government cells, and the 2025 " - "firm-size pretax-contribution offer rates. Values are copied as " - "published; null marks a cell AHRQ suppressed ('--'). Dollar " - "values are annual; percent values are percents." + "national rows that age the government cells, the 2025 national " + "share of enrollees whose coverage required no employee " + "contribution, and the 2024 and 2025 national private-sector " + "enrollment rows. Values are copied as published; null marks a " + "cell AHRQ suppressed ('--'). Dollar values are annual; percent " + "values are percents; employee counts are persons." ), "generator": "tools/build_us_meps_ic_esi_cells.py", "extraction": { @@ -508,9 +539,13 @@ def build_payload(pdf_dir: Path) -> dict: "private_state_2025": private_2025, "private_national_2024": private_2024_national, "public_division_2024": public_2024, - "pretax_contribution_2025": _pretax_offer_shares( - texts["private_national_2025"], PDFS[3] + "no_contribution_share_2025": _no_contribution_shares( + texts["private_state_2025"], PDFS[0] ), + "private_enrollment_national": { + "2024": _private_enrollment(texts["private_state_2024"], PDFS[1]), + "2025": _private_enrollment(texts["private_state_2025"], PDFS[0]), + }, } diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 12c8e968a1..4a646f724b 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -89,9 +89,6 @@ "pre_subsidy_care_expenses", "is_incapable_of_self_care", "is_spm_independent_minor_role", - # PolicyEngine/microcosm#454: the employee section 125 premium leaf the - # meps_esi_premiums base stage writes (the reference eCPS never had one). - "pre_tax_health_insurance_premiums", # PolicyEngine/microcosm#978 option 1: the ASEC reported-receipt inputs the # ACS local-area transfer requires in its donor. "receives_wic", @@ -121,24 +118,14 @@ "(PolicyEngine/microcosm#454), written by the meps_esi_premiums " "base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / " "NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person " - "files) and MEPS-IC tier x firm-size x State cells, scaled to BEA NIPA " - "7.8 line 17. A release must persist it: CBO household market income " - "(gov.household.cbo_market_income_additions) and every ESI-to-wages " - "reform bind through it. Required with NO reviewed exclusion; absent " - "on pre-#454 bases by construction — the intended red gate until a " - "base built with the stage ships. The release also holds its total to " - "the anchor (us_esi_premiums_anchor_gate)." - ), - "pre_tax_health_insurance_premiums": ( - "Employee premium paid through section 125 payroll deduction " - "(PolicyEngine/microcosm#454), written by the same meps_esi_premiums " - "stage: an employed policyholder with wages, a reported premium " - "(PHIP_VAL) and an employee share pays it pre-tax at the MEPS-IC 2025 " - "firm-size pretax-offer rate. PolicyEngine-US 2.2.1 subtracts it from " - "income-tax and FICA wages (gov.irs.gross_income.pre_tax_contributions, " - "fica_pre_tax_contributions); absent, both wage bases overstate. " - "Required with NO reviewed exclusion; absent on pre-#454 bases by " - "construction." + "files) and MEPS-IC tier x firm-size x State cells, scaled over every " + "policyholder to CMS NHE Table 24. A release must persist it: CBO " + "household market income (gov.household.cbo_market_income_additions) " + "and every ESI-to-wages reform bind through it. Required with NO " + "reviewed exclusion; absent on pre-#454 bases by construction — the " + "intended red gate until a base built with the stage ships. The " + "release also holds the anchor-universe total to the anchor " + "(us_esi_premiums_anchor_gate)." ), "schedule_d_capital_gain_distributions": ( "Schedule D line 13 route leg of the #282 capital-gain-distributions " @@ -393,36 +380,13 @@ "to CBO household market income through " "gov.household.cbo_market_income_additions, its only consumer, so " "neutralizing it lowers cbo_household_market_income by the " - "weighted employer-premium total: about $977 billion at 2024 " - "(BEA NIPA 7.8 line 17, the meps_esi_premiums anchor; the stage " - "measured $977.0 billion over 74.0 million weighted employed " - "policyholders on the default 2023-2025 pool, before calibration). " - "The floor sits at about half of that. A structural zero means " - "the column was dropped or the stage never ran." - ), - "issue": "PolicyEngine/microcosm#454", - }, - { - "id": "pre_tax_health_insurance_premium_neutralization", - "name": "Pre-tax health insurance premium neutralization", - "parameter_changes": {}, - "neutralized_variable": "pre_tax_health_insurance_premiums", - "budget_measure": "income_tax", - "period": 2024, - "effect_direction": "baseline_minus_reform", - "expected_sign": "negative", - "binding_inputs": ["pre_tax_health_insurance_premiums"], - "min_abs_effect": 5_000_000_000.0, - "reason": ( - "PolicyEngine-US 2.2.1 subtracts pre_tax_health_insurance_premiums " - "from employment_income in irs_employment_income (through " - "gov.irs.gross_income.pre_tax_contributions), so neutralizing it " - "raises taxable wages and income tax: baseline-minus-reform is " - "negative. The meps_esi_premiums stage measured $181.1 billion of " - "pre-tax premiums over 52.5 million weighted workers on the default " - "2023-2025 pool; at a typical 10-22 percent marginal rate that is " - "roughly $18-40 billion, so a $5 billion floor separates a " - "structural zero (the column dropped) from any plausible effect." + "weighted employer-premium total: about $926 billion at 2024 (the " + "employed policyholders' part of CMS NHE Table 24's $1,047.0 " + "billion, the meps_esi_premiums anchor; the stage measured $925.7 " + "billion over 74.0 million weighted employed policyholders on the " + "default 2023-2025 pool, before calibration). The floor sits at " + "about half of that. A structural zero means the column was " + "dropped or the stage never ran." ), "issue": "PolicyEngine/microcosm#454", }, @@ -1706,9 +1670,7 @@ def build_manifest() -> dict: "(PolicyEngine/microcosm#978 option 1), and the engine's declared " "dataset source input is_spm_independent_minor_role (the " "spm_independence_role base-builder stage; " - "PolicyEngine/microcosm#893), and the employee section 125 " - "premium leaf pre_tax_health_insurance_premiums (the " - "meps_esi_premiums base-builder stage; PolicyEngine/microcosm#454). " + "PolicyEngine/microcosm#893). " "status='reviewed_exclusion' for ecps_parity_known_gaps.json entries " "(reason+issue from that register); EXCEPT every primary-source " "restoration pinned by RESTORED_REFERENCE_ECPS_REQUIRED_INPUTS " diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index c5040ef43a..c41b1fdbff 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -241,23 +241,21 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: _ESI_ANCHOR_EXCLUSION = ( "deferred", - "NHE Table 24 employer contribution to ESI premiums. The pe-us input it " - "anchors (employer_sponsored_insurance_premiums) is now produced by the " - "meps_esi_premiums base stage (microcosm#454) for employed current ESI " - "policyholders only, while NHE counts COBRA and retiree enrollees (NHEA " - "methodology paper 2024, p. 28; Table 24 note 1 adds Medicare Retiree " - "Drug Subsidy payments), so a calibration target on this fact would load " - "the retiree share onto workers. Validation-gated instead: " - "us_esi_premiums_anchor_gate holds the calibrated total within 5% of BEA " - "NIPA 7.8 line 17 (the us-data target's concept, pinned in the stage) and " - "at or below this all-employer fact; the private-employer subset rides as " - "the sector cross-check.", - "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: MEPS-IC " - "cells over all 92.4M weighted current policyholders give $1,031.0B " - "(0.985 x NHE $1,047.0B CY2024); the employed universe gives $912.8B raw, " - "scaled to BEA $977.0B (11.5% of the all-policyholder total is " - "non-employed or no-employer coverage); private employers $756.7B vs NHE " - "private $753.4B", + "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums " + "base stage (microcosm#454) scales to this fact over the fact's own " + "universe, every current ESI policyholder (NHEA methodology paper 2024, " + "p. 29: active employees, COBRA and retirees), and writes the pe-us input " + "(employer_sponsored_insurance_premiums) for employed policyholders only. " + "So the fact is not a sum target on the column: the column carries about " + "88% of it and the rest is priced, not exported. Validation-gated " + "instead: us_esi_premiums_anchor_gate recomputes the anchor-universe " + "total at release weights and holds it within 5% of this all-employer " + "fact; the private-employer subset rides as a sector cross-check.", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M " + "weighted current policyholders; anchor-universe total $1,047.0B at scale " + "factor 1.040; employed column $925.7B (88.4%); other policyholders " + "$121.3B; private-sector part of the column $713.2B against MEPS-IC's own " + "private active-employee total of $668.6B (2024)", _fence( origin=( "a REAL us-data calibration target: utils/loss.py pinned " @@ -274,19 +272,20 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: ), purpose=( "anchor the CBO market-income ESI component once the column has a " - "live producer. NIPA 7.8 line 17 counts employer contributions as " - "compensation of current employees (BEA builds it from MEPS-IC " - "'insurance purchased by employers for employees'); NHE Table 24 " - "adds COBRA and retiree enrollees, so it bounds the column from " - "above (7.2% above BEA in CY2024)." + "live producer. NHE Table 24 and NIPA 7.8 line 17 both count " + "employer contributions for active and retired employees (NHEA " + "methodology paper; BEA State Personal Income methodology, " + "December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in " + "CY2024 and is recorded beside the gate verdict." ), verdict_basis=( - "deferred, not dropped: the producer is live, but the only banked " - "fact measures a broader concept than the column. Release " - "validation (us_esi_premiums_anchor_gate, both dense and sparse " - "exports) uses it as the upper bound today; bank BEA NIPA 7.8 line " - "17 (B4923C) in the ledger and compile that as the calibration " - "target, and this family stays the cross-check." + "deferred, not dropped: the producer is live and scaled to this " + "fact, but the column is the employed policyholders' part of it, " + "so the fact cannot be compiled as a sum target on the column. " + "Release validation (us_esi_premiums_anchor_gate, both dense and " + "sparse exports) holds the anchor-universe total to it. A " + "calibration target needs an exported anchor-universe quantity or " + "a banked active-employee fact." ), ), ) @@ -557,15 +556,18 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: _fence( origin="us-data loss.py employer_sponsored_insurance_premiums target", purpose=( - "anchor employer-paid ESI premiums for current employees: the " - "compensation concept the employed-policyholder column carries." + "anchor employer-paid ESI premiums. The series counts " + "contributions for active and retired employees (BEA State " + "Personal Income methodology, December 2025, paragraphs " + "3.18-3.19), the same concept as NHE Table 24." ), verdict_basis=( "source-absent from the feed: the meps_esi_premiums stage pins " "the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., " - "published 2026-09-30: $977.034B CY2024) and scales to it, and " - "the release anchor gate validates against it; compile it here " - "once the ledger banks the series." + "published 2026-09-30: $977.034B CY2024, revised from " + "$1,002.9B) and records the anchor-universe total against it " + "beside the release anchor verdict; the stage scales to NHE " + "Table 24. Compile it here once the ledger banks the series." ), ), ), diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 19bd607938..70ed074fb8 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "0ce29ce06285a094b214f805a8136f399216aae44dc4b55d13a568307f5c0f91" + "e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index a58034dc2a..d4b4e88283 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "8afbe29a5ad75630b11e561e242d1882c94efc10648d287e80c6ab0a7f44f1bc" + "e8d32da2dbdaa015cb4eb363084c66cf509a93dde76cd473113750120dcaef80" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] diff --git a/tools/us_bundle_generation/identity_contracts.py b/tools/us_bundle_generation/identity_contracts.py index c0d48e811e..06b2bf04d8 100644 --- a/tools/us_bundle_generation/identity_contracts.py +++ b/tools/us_bundle_generation/identity_contracts.py @@ -102,7 +102,7 @@ def _stages_with_operation( def build_seed_site_bindings( source_document: Mapping[str, Any], ) -> list[dict[str, object]]: - """Bind all 54 legacy-v1 sites to their concrete execution owners.""" + """Bind all 53 legacy-v1 sites to their concrete execution owners.""" source_ids = _source_stage_ids(source_document) transfer_nodes = tuple(group.name for group in CANONICAL_US_LATE_TRANSFER_GROUPS) @@ -169,7 +169,6 @@ def build_seed_site_bindings( ), "snap_take_up_assignment": source("snap_take_up"), "pregnancy_assignment": source("pregnancy"), - "pre_tax_health_premium_assignment": source("meps_esi_premiums"), "wic_claim_assignment": source("wic_claim_input"), "snap_discretionary_exemption_assignment": source( "snap_abawd_discretionary_exemption" @@ -204,9 +203,9 @@ def build_seed_site_bindings( } protocol_site_ids = tuple(site.id for site in LEGACY_V1_PROTOCOL.sites) - if len(protocol_site_ids) != 54 or set(owners) != set(protocol_site_ids): + if len(protocol_site_ids) != 53 or set(owners) != set(protocol_site_ids): raise RuntimeError( - "legacy-v1 seed owner ledger must cover exactly 54 protocol sites; " + "legacy-v1 seed owner ledger must cover exactly 53 protocol sites; " f"missing={sorted(set(protocol_site_ids) - owners.keys())!r}, " f"extra={sorted(owners.keys() - set(protocol_site_ids))!r}" ) From 4110519163612fc72c1c27cd3164b657bfcdcbef Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 19:58:03 -0400 Subject: [PATCH 5/8] Re-pin three test counts for the single-output ESI stage (#454) The reform-probe count drops to 42 with the pre-tax probe gone, the pool tool's import graph reaches esi_premiums through the operator boundary (75 modules), and the ESI stage no longer reads PHIP_VAL. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/us/test_us_other_health_insurance.py | 4 +++- .../engine_free/us/test_us_post_export_scoring.py | 6 +++--- .../tests/engine_free/us/test_us_spine_blindness.py | 10 ++++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py index 2a65e9476a..8fdc9ed67d 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_other_health_insurance.py @@ -591,8 +591,10 @@ def test_esi_employer_premium_is_owned_by_its_own_stage_not_proxied_here() -> No "NOW_HIPAID", "NOW_GRPFTYP", "NOW_GRPFTYP2", - "PHIP_VAL", } <= source_columns + # The employer premium never reads the reported premium: PHIP_VAL stays + # this stage's source alone. + assert "PHIP_VAL" not in source_columns forbidden_proxies = {"has_esi", "NOW_GRP", "is_esi_dependent", "tax_unit_size"} assert forbidden_proxies.isdisjoint(source_columns) assert not any( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py index c603c42d33..2d13558233 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_post_export_scoring.py @@ -75,7 +75,7 @@ def test_batched_engines_partition_release_and_order_their_work( validation = consumers["reform_validation"].record() demographics = consumers["demographics"].record() probes = len(us_release_reform_coverage_probes()) - assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 43 + assert smoke["reform_systems"] == smoke["reform_passes"] == probes == 42 assert validation["reform_systems"] == validation["reform_passes"] > 0 assert demographics["reform_systems"] == demographics["reform_passes"] == 0 assert all(engine.reform is None for engine in constructions) @@ -1377,7 +1377,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( ) -> None: sweep = _load_sweep_module() plans, requests, counts = sweep._record_requests(builder) - assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 43 + assert counts["smoke_probes"] == len(us_release_reform_coverage_probes()) == 42 specs = reform_validation_module.load_default_reform_specs(period=builder.PERIOD) assert counts["validation_specs"] == sum(not spec.in_sample for spec in specs) == 52 assert ( @@ -1421,7 +1421,7 @@ def test_guard_sweep_records_and_scores_shipped_reform_passes( assert ( len(requests) == counts["smoke_probes"] + counts["validation_reform_passes"] - == 109 + == 108 ) frame, log = _nested_frame(), _EngineLog() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py index da135839f6..5cd4c95992 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_spine_blindness.py @@ -3487,15 +3487,17 @@ def test_pool_build_tool_import_graph_is_source_spine_blind() -> None: for tool in _SPINE_BLIND_BUILD_TOOLS: runtime_graph, missing_modules = _us_runtime_import_graph(tool) - # 74 = main's 70 plus spm_independence_role.py, spm_role_source.py and + # 75 = main's 70 plus spm_independence_role.py, spm_role_source.py and # spm_composition.py, reached because the pool's engine-input # projection names the SPM role as a required source input (#893), # plus asec_sources.py, whose default ASEC pool the education sidecar - # loader now defaults to. All four are classified in + # loader now defaults to, plus esi_premiums.py, whose output family + # operator_boundary registers so a raw source frame that already + # carries it is refused (#454). All five are classified in # _OTHER_US_RUNTIME_MODULES and scanned below like every other # reached module. - assert len(runtime_graph) == 74, ( - f"{tool.name} must reach the pinned 74-module runtime graph; " + assert len(runtime_graph) == 75, ( + f"{tool.name} must reach the pinned 75-module runtime graph; " f"reached {len(runtime_graph)}" ) assert not missing_modules, ( From 77adf4d9e7a6dab2b890e1c55f33da0153e75517 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 10 Oct 2026 21:09:30 -0400 Subject: [PATCH 6/8] Answer the second review's non-blocking findings on the ESI stage (#454) The second independent review approved with nits and no blocking findings. - Target parity: the NHE private-employer family no longer borrows the all-employer text. Nothing holds the private part of the column to that fact; it is recorded against MEPS-IC's active-employee total. - L0/refit export: the ESI column joins the required person source columns, so the standalone export refuses a frame where it is absent or constant. The doc says that path still runs neither ESI gate. - Doc: NOEMP is the employer of last year's longest job while PEIO1COW is the current job; an unmeasured "most" is removed; the unincorporated self-employed are priced and left out; the NHE digest is of the zip archive. - Cell table: --check compares published values, titles and pins and no longer depends on the installed poppler version. - Tests: every tier x State x employer cell reconciles to its published mean under the published mix; the anchor gate is exercised without each raw column and without the household State. The spec bundle is unchanged. Co-Authored-By: Claude Opus 5.5 --- docs/us-esi-employer-premiums.md | 25 ++++++-- .../build/us/target_parity_manifest.json | 6 +- .../data/MEPS_IC_ESI_PREMIUM_CELLS.md | 4 +- .../build/us_runtime/esi_premiums.py | 2 + .../build/us_runtime/l0_refit_export.py | 4 ++ .../engine_free/us/test_us_esi_premiums.py | 62 ++++++++++++++++++- .../engine_free/us/test_us_l0_refit_export.py | 32 ++++++++++ tools/build_us_acs_local_release.py | 4 +- tools/build_us_fiscal_refresh_release.py | 2 +- tools/build_us_meps_ic_esi_cells.py | 29 ++++++++- tools/build_us_target_parity_manifest.py | 23 ++++++- 11 files changed, 173 insertions(+), 20 deletions(-) diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md index d1f2e30cad..886d34ac05 100644 --- a/docs/us-esi-employer-premiums.md +++ b/docs/us-esi-employer-premiums.md @@ -80,7 +80,9 @@ Everyone else carries zero in the column: - retired, unemployed and other non-employed policyholders, because their coverage is not compensation of a current job; - the unincorporated self-employed and workers without pay (`PEIO1COW` 7 and - 8), who have no employer to pay a share. + 8), whose current job is not an employer's job. Their group coverage is + priced in the anchor universe, like a retiree's, and left out of the + column. The non-employed policyholders' scaled share is priced and then left out. It is not loaded onto workers. @@ -100,9 +102,15 @@ employee, by coverage tier: - Local and federal government (`PEIO1COW` 3 and 1): the same table's all-governments column. MEPS-IC does not survey the federal government; this is a stand-in. -- No employer class (`PEIO1COW` 0, 7 or 8, which is most non-employed - policyholders): the State's all-sizes private column. A non-employed - policyholder who does report a class of worker keeps that employer's cell. +- No employer class (`PEIO1COW` 0, 7 or 8): the State's all-sizes private + column. A non-employed policyholder who does report a class of worker keeps + that employer's cell. + +`NOEMP` and `PEIO1COW` describe different jobs. `PEIO1COW` is the class of +worker of the current job. `NOEMP` is the size of the employer of the longest +job held last year. A worker who changed jobs takes the earlier employer's +size band, and a current worker with no job last year (`NOEMP` 0) takes the +State's all-sizes column. Series III for 2025 is not yet published. The 2024 government cells are aged to 2025 by the private-sector national ratio of the same tier and measure @@ -283,6 +291,7 @@ the PUF-support clone, and the clone conserves both totals. | `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Column present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible; no employer premium outside the column universe or where the employer pays none; every premium is one common multiple of its MEPS-IC share. | | `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of NHE Table 24 for the release period. Employed policyholders must carry 80% to 95% of it. | | `us_release_input_coverage_gate` | release tool | The input is `required` with no reviewed exclusion, so an export that drops or flattens it fails. | +| `assert_required_us_release_source_columns` | the L0/refit export (`l0_refit_export.export_us_l0_refit_h5`, run by `tools/export_us_l0_refit_h5.py`), unless `--allow-missing-source-columns` is passed | The column must be present and non-constant. | | reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B. | What the gates need, and what they do not catch: @@ -320,7 +329,8 @@ stage; the failures it waives are still written to that file. `us_runtime/data/meps_ic_esi_premium_cells.json`, whose own SHA-256 the stage pins and checks on load. - NHE and BEA files: URL, byte length and SHA-256 in the stage manifest - (`us/source_stages.json`), and URL and SHA-256 in the module. + (`us/source_stages.json`), and URL and SHA-256 in the module. The NHE + digest is of the zip archive, not of the Table 24 workbook inside it. - The base summary records the stage's full summary twice (`esi_premiums_signal`, `esi_premiums_post_clone_signal`): scale factor, raw and scaled totals for both universes, totals by tier and employer sector, @@ -342,4 +352,9 @@ stage; the failures it waives are still written to that file. so the fact cannot be compiled as a sum target on it. The target-parity manifest keeps the two `cms_nhe` ESI families as `deferred` reviewed exclusions and records BEA's series as a source-absent family. +- **The standalone L0/refit export.** `tools/export_us_l0_refit_h5.py` + writes an H5 without running the signal or anchor gate. By default it + refuses a frame whose column is absent or constant, and nothing more: the + two ESI gates run only in the base build and in + `tools/build_us_fiscal_refresh_release.py`. - **The pre-tax employee premium.** See the section above. diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 914c39fe76..f419dcfc5e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -460,7 +460,7 @@ "cms_nhe.esi_employer_contribution_premiums": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums base stage (microcosm#454) scales to this fact over the fact's own universe, every current ESI policyholder (NHEA methodology paper 2024, p. 29: active employees, COBRA and retirees), and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this all-employer fact; the private-employer subset rides as a sector cross-check.", + "reason": "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums base stage (microcosm#454) scales to this fact over the fact's own universe, every current ESI policyholder (NHEA methodology paper 2024, p. 29: active employees, COBRA and retirees), and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this all-employer fact.", "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $1,047.0B at scale factor 1.040; employed column $925.7B (88.4%); other policyholders $121.3B; private-sector part of the column $713.2B against MEPS-IC's own private active-employee total of $668.6B (2024)", "fence": { "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", @@ -471,8 +471,8 @@ "cms_nhe.esi_private_employer_contribution_premiums": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums base stage (microcosm#454) scales to this fact over the fact's own universe, every current ESI policyholder (NHEA methodology paper 2024, p. 29: active employees, COBRA and retirees), and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this all-employer fact; the private-employer subset rides as a sector cross-check.", - "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $1,047.0B at scale factor 1.040; employed column $925.7B (88.4%); other policyholders $121.3B; private-sector part of the column $713.2B against MEPS-IC's own private active-employee total of $668.6B (2024)", + "reason": "NHE Table 24 private-employer contribution to ESI premiums, the private part of the all-employer fact the meps_esi_premiums stage scales to (microcosm#454). It is neither compiled nor gated: the stage scales one factor to the all-employer total over every current ESI policyholder, and nothing holds the private-sector part of the column to this fact, which also counts private retirees and COBRA enrollees. The private part is instead recorded beside the anchor verdict against MEPS-IC's own active-employee total (us_esi_premiums_anchor_gate, private_active_cross_check).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: private-sector part of the column $713.2B; MEPS-IC private active-employee total $668.6B (2024)", "fence": { "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NHE Table 24 and NIPA 7.8 line 17 both count employer contributions for active and retired employees (NHEA methodology paper; BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in CY2024 and is recorded beside the gate verdict.", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md index f710dac2d3..926a3ae5f3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/data/MEPS_IC_ESI_PREMIUM_CELLS.md @@ -36,7 +36,9 @@ year, and a United States row that differs from the AHRQ national figure it must reproduce (for example single $9,025 and family $26,281 in 2025; the 2024 rows of MEPS-IC Research Findings #54). A national row whose kept column is suppressed or flagged unreliable is refused too. `--check` regenerates in -memory and fails if the committed file differs. +memory and fails if any published value, title or pin differs; it ignores the +`pdftotext` version and the `text_line` positions, which depend on the +installed poppler. The stage pins this file's own SHA-256 (`esi_premiums._CELLS_SHA256`) and refuses to load any other bytes, so a regeneration must be re-pinned there diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py index dc2c722a97..10b17df110 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py @@ -202,6 +202,8 @@ #: private, federal and State and local employers, for active employees, COBRA #: enrollees and retirees. The anchor microcosm#454 chose; banked in the #: pinned feed as cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums. +#: ``sha256`` is the digest of the zip archive at ``source`` (520,391 bytes), +#: not of the Table 24 workbook inside it. EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( { "source": _NHE_TABLES_URL, diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py b/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py index ce76b43846..10cd209e3d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/l0_refit_export.py @@ -47,6 +47,9 @@ US_ELIGIBILITY_INPUTS_NONCONSTANT_PERSON_COLUMNS, ) from microcosm.build.us_runtime.energy_subsidy import US_ENERGY_SUBSIDY_OUTPUT_COLUMNS +from microcosm.build.us_runtime.esi_premiums import ( + US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, +) from microcosm.build.us_runtime.farm_business_income import ( US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, ) @@ -189,6 +192,7 @@ *US_QBI_NONCONSTANT_PERSON_COLUMNS, *US_ORG_WAGES_NONCONSTANT_PERSON_COLUMNS, *US_OTHER_HEALTH_INSURANCE_NONCONSTANT_PERSON_COLUMNS, + *US_ESI_PREMIUMS_NONCONSTANT_PERSON_COLUMNS, *US_SE_HEALTH_ATTRIBUTION_OUTPUT_COLUMNS, *US_ADULT_CARE_OUTPUT_COLUMNS, *US_FARM_BUSINESS_INCOME_NONCONSTANT_PERSON_COLUMNS, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py index 5e7ec72b35..fb06908cfc 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py @@ -214,7 +214,7 @@ def _realistic_rows(count: int = 400) -> list[dict]: class TestManifestDeclaration: - def test_stage_declares_both_outputs_and_the_reviewed_operations(self) -> None: + def test_stage_declares_its_output_and_the_reviewed_operations(self) -> None: spec = us_esi_premiums_stage_spec() assert spec.stage == US_ESI_PREMIUMS_STAGE_NAME == "meps_esi_premiums" assert tuple(spec.outputs) == US_ESI_PREMIUMS_OUTPUT_COLUMNS @@ -485,6 +485,45 @@ def test_the_published_mix_reproduces_the_published_employer_mean( modeled = _column_total(result) / summary["scale_factor"] assert modeled == pytest.approx(published) + def test_every_cell_reconciles_under_the_published_mix(self) -> None: + """Exhaustive: every tier x State x size private cell and every + tier x division x column government cell. + + For each cell, the no-contribution share of enrollees paying nothing + and the rest paying the conditional contribution average the cell's + premium less its average contribution. A suppressed contribution cell + reconciles to its fallback value, since AHRQ published none. + """ + + rows = [] + tiers = ((1, 1), (2, 1), (3, 2)) # NOW_GRPFTYP2, NOW_GRPFTYP + for tier_code, grpftyp in tiers: + for state in _STATES: + # Private by size (NOEMP 0, 1, 3), State and local government. + for sector, noemp in ((4, 0), (4, 1), (4, 3), (2, 0), (3, 0), (1, 0)): + for hipaid in (1, 2): + rows.append( + { + "NOW_GRPFTYP2": tier_code, + "NOW_GRPFTYP": grpftyp, + "NOW_HIPAID": hipaid, + "PEIO1COW": sector, + "NOEMP": noemp, + "state_fips": state, + } + ) + person = esi._person_with_state(_frame(rows)) + codes, raw, premium, contribution = esi._person_raw_shares(person) + _, _, share = esi._cell_values(codes, _CELLS) + pays_all, pays_some = raw[0::2], raw[1::2] + assert (codes.hipaid[0::2] == 1).all() and (codes.hipaid[1::2] == 2).all() + unpaid = share[0::2] + modeled = unpaid * pays_all + (1 - unpaid) * pays_some + published = premium[0::2] - contribution[0::2] + assert len(modeled) == 3 * 51 * 6 + assert (pays_some > 0).all() and (pays_some < pays_all).all() + np.testing.assert_allclose(modeled, published, rtol=1e-12) + def test_the_unconditional_contribution_would_overstate_the_cell_mean(self) -> None: # The national under-50 single-coverage cell: premium $9,034, average # contribution $1,924, 35.5% of enrollees pay nothing. Subtracting the @@ -999,7 +1038,10 @@ def test_a_zero_mass_column_is_red(self, column) -> None: assert not gate.passed assert f"{column}: zero weighted mass." in gate.failures - @pytest.mark.parametrize("column", ["NOW_OWNGRP", "NOW_HIPAID", "PEMLR", "NOEMP"]) + @pytest.mark.parametrize( + "column", + [c for c in US_ESI_PREMIUMS_REQUIRED_SOURCE_COLUMNS if c != "state_fips"], + ) def test_an_export_without_a_raw_column_is_red(self, column) -> None: # The anchor counts every policyholder, so the gate cannot grade a # frame that no longer says who they are. @@ -1010,6 +1052,20 @@ def test_an_export_without_a_raw_column_is_red(self, column) -> None: assert any("assignment provenance incomplete" in f for f in gate.failures) assert "anchor_universe_employer_total" not in gate.details + def test_a_frame_without_the_household_state_fails_both_gates(self) -> None: + # State sits on the household table; without it no cell can be read. + frame = self._export() + frame.table("household").drop(columns=["state_fips"], inplace=True) + for gate in ( + us_esi_premiums_signal_gate(frame), + us_esi_premiums_anchor_gate(frame, time_period=TIME_PERIOD), + ): + assert not gate.passed + assert any( + "assignment provenance incomplete" in f and "state_fips" in f + for f in gate.failures + ) + @pytest.mark.parametrize("year", [2025, 2031]) def test_a_year_without_an_anchor_is_red(self, year) -> None: gate = us_esi_premiums_anchor_gate(self._export(), time_period=year) @@ -1329,7 +1385,7 @@ def test_pricing_other_policyholders_never_raises_a_workers_premium(population) st.sampled_from(_STATES), st.floats(0.5, 500.0, allow_nan=False), ) -def test_the_published_mix_reconciles_every_private_cell( +def test_the_published_mix_reconciles_sampled_private_cells_through_the_stage( tier_code, noemp, sector, state, mass ) -> None: tier = esi._TIER_BY_CODE[tier_code] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py index ce708331ae..628538479f 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_l0_refit_export.py @@ -59,6 +59,7 @@ def _us_frame(**person_extra: object) -> Frame: "takes_up_wic_if_eligible": [True, False, True], "educator_expense": [0.0, 300.0, 150.0], "other_health_insurance_premiums": [1_200.0, 0.0, 600.0], + "employer_sponsored_insurance_premiums": [9_400.0, 0.0, 6_100.0], "health_insurance_premiums": [1_200.0, 0.0, 800.0], "is_self_employed": [True, False, True], "is_incapable_of_self_care": [False, True, False], @@ -908,6 +909,37 @@ def test_required_us_release_source_columns_enforces_educator_expense_signal() - assert_required_us_release_source_columns(raw_frame) +def test_required_us_release_source_columns_enforces_esi_premium_signal() -> None: + """microcosm#454: an L0/refit export refuses an absent or flat ESI column.""" + + frame = _us_frame() + for mutate, message in ( + ( + lambda people: people.drop( + columns=["employer_sponsored_insurance_premiums"] + ), + "person.employer_sponsored_insurance_premiums: missing", + ), + ( + lambda people: people.assign(employer_sponsored_insurance_premiums=0.0), + "person.employer_sponsored_insurance_premiums: not nonconstant", + ), + ): + raw_frame = Frame( + { + **{ + entity: frame.table(entity).copy() + for entity in frame.schema.entities + }, + "person": mutate(frame.table("person").copy()), + }, + frame.schema, + {"household": frame.weights_for("household")}, + ) + with pytest.raises(ValueError, match=message): + assert_required_us_release_source_columns(raw_frame) + + def test_required_us_release_source_columns_enforces_other_premium_signal() -> None: frame = _us_frame() raw_people = frame.table("person").copy() diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index a9f430dfe5..eaca029f7a 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -503,11 +503,11 @@ def fill_reviewed_nulls( Every registered (entity, column) has its NaN filled with the pe-us variable's own default; NaN in any engine-input column NOT in the register is a hard error with a per-spine diagnostic — an artifact - defect, surfaced not filled. The two ESI premium inputs are never + defect, surfaced not filled. The ESI employer premium input is never filled, registered or not: a null there refuses the build. """ - # microcosm#454: the ESI premium inputs exist only where the + # microcosm#454: the ESI employer premium exists only where the # meps_esi_premiums stage ran (the ASEC donor rows). Refuse before any # fill: a registered null here would become a silent zero on the ACS spine. refuse_unassigned_us_esi_premiums( diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 3214e4da4e..742618d337 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -15369,7 +15369,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: ) # microcosm#454: the calibrated ESI premium total over the anchor's # universe (every policyholder) against CMS NHE Table 24 within tolerance, - # red when either column is absent or zero-mass or the raw ASEC coverage + # red when the column is absent or zero-mass or the raw ASEC coverage # columns are gone. Runs on the calibrated export for BOTH the dense and # sparse default paths; the verdict ships as the esi_premiums gate # evidence, with the base and export signal verdicts beside it so a waived diff --git a/tools/build_us_meps_ic_esi_cells.py b/tools/build_us_meps_ic_esi_cells.py index ef1126c93c..137ce487c8 100644 --- a/tools/build_us_meps_ic_esi_cells.py +++ b/tools/build_us_meps_ic_esi_cells.py @@ -26,7 +26,9 @@ uv run python tools/build_us_meps_ic_esi_cells.py --pdf-dir /tmp/meps-ic -``--check`` regenerates in memory and fails if the committed file differs. +``--check`` regenerates in memory and fails if any published value, title or +pin in the committed file differs. It ignores the ``pdftotext`` version string +and the ``text_line`` positions, which depend on the installed poppler. """ from __future__ import annotations @@ -553,6 +555,25 @@ def render(payload: dict) -> str: return json.dumps(payload, indent=1, sort_keys=True) + "\n" +#: Provenance that depends on the installed poppler, not on the PDFs: the +#: ``pdftotext`` version string and the text line each table was found on. +_ENVIRONMENT_KEYS = frozenset({"pdftotext", "text_line"}) + + +def published_values(payload): + """The payload without poppler-dependent provenance, for ``--check``.""" + + if isinstance(payload, dict): + return { + key: published_values(value) + for key, value in payload.items() + if key not in _ENVIRONMENT_KEYS + } + if isinstance(payload, list): + return [published_values(item) for item in payload] + return payload + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( @@ -567,9 +588,11 @@ def main(argv: list[str] | None = None) -> int: help="Fail if the committed cell table differs from a regeneration.", ) args = parser.parse_args(argv) - rendered = render(build_payload(args.pdf_dir)) + payload = build_payload(args.pdf_dir) + rendered = render(payload) if args.check: - if OUTPUT_PATH.read_text() != rendered: + committed = json.loads(OUTPUT_PATH.read_text()) + if published_values(committed) != published_values(payload): print(f"{OUTPUT_PATH} is stale; regenerate it.", file=sys.stderr) return 1 print(f"{OUTPUT_PATH} is current.") diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index c41b1fdbff..e53a2c460f 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -250,7 +250,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "88% of it and the rest is priced, not exported. Validation-gated " "instead: us_esi_premiums_anchor_gate recomputes the anchor-universe " "total at release weights and holds it within 5% of this all-employer " - "fact; the private-employer subset rides as a sector cross-check.", + "fact.", "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M " "weighted current policyholders; anchor-universe total $1,047.0B at scale " "factor 1.040; employed column $925.7B (88.4%); other policyholders " @@ -290,9 +290,28 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: ), ) +_ESI_PRIVATE_EMPLOYER_EXCLUSION = ( + "deferred", + "NHE Table 24 private-employer contribution to ESI premiums, the private " + "part of the all-employer fact the meps_esi_premiums stage scales to " + "(microcosm#454). It is neither compiled nor gated: the stage scales one " + "factor to the all-employer total over every current ESI policyholder, " + "and nothing holds the private-sector part of the column to this fact, " + "which also counts private retirees and COBRA enrollees. The private part " + "is instead recorded beside the anchor verdict against MEPS-IC's own " + "active-employee total (us_esi_premiums_anchor_gate, " + "private_active_cross_check).", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: " + "private-sector part of the column $713.2B; MEPS-IC private " + "active-employee total $668.6B (2024)", + _ESI_ANCHOR_EXCLUSION[3], +) + _FAMILY_EXCLUSIONS: dict[str, tuple[str, str, str, dict[str, str]]] = { "cms_nhe.esi_employer_contribution_premiums": _ESI_ANCHOR_EXCLUSION, - "cms_nhe.esi_private_employer_contribution_premiums": _ESI_ANCHOR_EXCLUSION, + "cms_nhe.esi_private_employer_contribution_premiums": ( + _ESI_PRIVATE_EMPLOYER_EXCLUSION + ), "jct.obbba_title_vii": _JCT_OBBBA_NO_TAX_ANCHOR_EXCLUSION, "bea_nipa.personal_interest_income": ( "macro_control_total", From 3cba5525499c327fac0009c3377f79911f6353be Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 17:40:46 -0400 Subject: [PATCH 7/8] Scale the ESI stage to BEA B4923C and bank it in the US feed (#454, d1317) Decision d1317 (2026-10-11) chose BEA NIPA Table 7.8 line 17 (B4923C) over CMS NHE Table 24 as the scale anchor, with the US Chronicle feed re-pinned to carry the banked fact. Stage. EMPLOYER_PREMIUM_ANCHOR is now B4923C for 2023 to 2025 ($923,195M, $977,034M, $1,027,929M, the 2026-09-30 vintage); NHE Table 24 becomes the recorded cross-check beside MEPS-IC's active-employee total. The universe design is unchanged: every current policyholder is priced and scaled, and the employed part is written. On the default pool the column is $863.9B (mean $11,679 per policyholder with a positive premium); its private part is 0.4% below MEPS-IC's own active-employee total. Feed. The pin moves from Chronicle 51aa40f to cf9ece2 (tag microcosm-us-feed-454-v1): 51aa40f plus the digit-text guard match from aa3e4de3 and the B4923C package as chronicle#331 merged it. All 40,178 previous rows are byte-identical, exactly three facts are added, all 33,250 compiled targets are identical, and two rebuilds give the same digest. The release tool refuses a feed whose B4923C values differ from the stage's anchor (us_esi_premiums_anchor_feed_mismatches). The family is a deferred reviewed exclusion, not a target (calibration is d1264). Also answers the round-3 review's nits: the NHE private-employer family has its own verdict basis, the NOEMP sentence is qualified, and the L0 export's generic input-mass parity check is disclosed. Co-Authored-By: Claude Opus 5.5 --- .../us-esi-employer-premiums-454.added.md | 2 +- docs/evidence/spec-engine/us-f0-coverage.json | 36 ++-- docs/us-chronicle-feed-repin.md | 62 ++++++- docs/us-esi-employer-premiums.md | 98 ++++++----- experiments/us-esi-454/README.md | 13 ++ .../us-esi-454/feed-repin/compare_feeds.py | 72 ++++++++ .../feed-repin/compare_registries.py | 73 ++++++++ .../us-esi-454/feed-repin/registry_diff.json | 7 + .../us-esi-454/feed-repin/value_diff.json | 46 +++++ .../receipts/stage_on_pool_2022_2024.json | 148 ++++++++-------- .../receipts/stage_on_pool_2023_2025.json | 150 ++++++++-------- .../us-esi-454/run_stage_on_pinned_pool.py | 8 +- .../build/spec_engine/field_usage.py | 8 +- .../build/spec_engine/inventory_coverage.py | 6 +- .../microcosm/build/us/chronicle_feed.json | 10 +- .../build/us/chronicle_feed_scope.json | 33 +++- .../us/release_input_coverage_manifest.json | 4 +- .../src/microcosm/build/us/source_stages.json | 37 ++-- .../src/microcosm/build/us/spec/sources.yaml | 63 +++---- .../build/us/target_parity_feed_families.json | 5 +- .../build/us/target_parity_manifest.json | 36 ++-- .../microcosm/build/us_runtime/__init__.py | 2 + .../build/us_runtime/esi_premiums.py | 153 +++++++++++------ .../us/test_spec_engine_coverage_tool.py | 10 +- .../shared/test_spec_engine_field_usage.py | 10 +- .../us/test_us_bundle_core_contracts.py | 2 +- .../engine_free/us/test_us_chronicle_feed.py | 11 +- .../engine_free/us/test_us_esi_premiums.py | 133 ++++++++++----- .../us/test_us_fiscal_refresh_builder.py | 19 ++- .../us/test_us_multispine_pool_tool.py | 2 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_fiscal_refresh_release.py | 11 ++ ...uild_us_release_input_coverage_manifest.py | 8 +- tools/build_us_target_parity_manifest.py | 160 ++++++++++-------- tools/generate_us_bundle_from_constants.py | 2 +- tools/spec_engine_coverage.py | 2 +- 36 files changed, 949 insertions(+), 495 deletions(-) create mode 100644 experiments/us-esi-454/feed-repin/compare_feeds.py create mode 100644 experiments/us-esi-454/feed-repin/compare_registries.py create mode 100644 experiments/us-esi-454/feed-repin/registry_diff.json create mode 100644 experiments/us-esi-454/feed-repin/value_diff.json diff --git a/changelog.d/us-esi-employer-premiums-454.added.md b/changelog.d/us-esi-employer-premiums-454.added.md index 25434896b9..b6973abf21 100644 --- a/changelog.d/us-esi-employer-premiums-454.added.md +++ b/changelog.d/us-esi-employer-premiums-454.added.md @@ -1 +1 @@ -The US base build produces `employer_sponsored_insurance_premiums` (microcosm #454). The `meps_esi_premiums` stage, declared but unable to run because the pinned ASEC inputs lack the Census policyholder fields, now reads `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP2`, `PEMLR` and `NOEMP` restored from the pinned Census person files. Every current ESI policyholder takes the MEPS-IC 2025 employer share for their coverage tier, firm size and State (census division for government employers), and one factor scales that whole universe to CMS NHE Table 24's employer contribution ($1,047.0B for 2024), which counts active employees, COBRA enrollees and retirees. The column carries the employed policyholders' part ($925.7B on the default pool, before calibration); dependents, non-employed policyholders and the unincorporated self-employed carry zero. A release must now carry the input with signal and its raw ASEC coverage columns, and hold the anchor-universe total within 5% of the anchor on the dense and the sparse export; the verdict ships as the `esi_premiums` gate evidence. The multispine pool does not run the stage yet, and an ACS local build refuses a frame whose input is null on its ACS rows. `pre_tax_health_insurance_premiums` is not produced: it waits for the engine's disjoint premium contract (policyengine-us #10046). +The US base build produces `employer_sponsored_insurance_premiums` (microcosm #454). The `meps_esi_premiums` stage, declared but unable to run because the pinned ASEC inputs lack the Census policyholder fields, now reads `NOW_OWNGRP`, `NOW_HIPAID`, `NOW_GRPFTYP2`, `PEMLR` and `NOEMP` restored from the pinned Census person files. Every current ESI policyholder takes the MEPS-IC 2025 employer share for their coverage tier, firm size and State (census division for government employers), and one factor scales that whole universe to BEA NIPA Table 7.8 line 17 (B4923C, $977.0B for 2024), which counts contributions for active and retired employees (decision d1317). The column carries the employed policyholders' part ($863.9B on the default pool, before calibration); dependents, non-employed policyholders and the unincorporated self-employed carry zero. The US Chronicle feed is re-pinned to banked B4923C facts (Chronicle `cf9ece2`, three facts added, every other fact byte-identical), and the release tool refuses a feed whose values differ from the stage's. A release must carry the input with signal and its raw ASEC coverage columns, and hold the anchor-universe total within 5% of the anchor on the dense and the sparse export; the verdict ships as the `esi_premiums` gate evidence, with CMS NHE Table 24 and MEPS-IC's active-employee total as recorded cross-checks. The multispine pool does not run the stage yet, and an ACS local build refuses a frame whose input is null on its ACS rows. `pre_tax_health_insurance_premiums` is not produced: it waits for the engine's disjoint premium contract (policyengine-us #10046). diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index ac9a0a167c..286b7a096f 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,9 +4,9 @@ "version": 1 }, "country": "us", - "documentation_sha256": "f002c79cc4a9296d34573d9c259eea0058f06e999630752ae3396395333889eb", + "documentation_sha256": "58a745afcfd8c3fd55317a61c4e79880dffbaf49344888b28b331d0318e67404", "field_usage": { - "authored_normative_field_count": 32436, + "authored_normative_field_count": 32437, "claim_count": 49, "claims": [ { @@ -514,8 +514,8 @@ ], "mode": "legacy_behavior", "pointer_class": "all", - "pointer_count": 1748, - "pointer_sha256": "dcc891c12eb675ee7f21c903995db98c3b8c6ccba569d71f0fd59b589abce04c", + "pointer_count": 1749, + "pointer_sha256": "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", "rationale": null, "relative_sink_prefix": "/source_manifest/stages", "source_prefix": "/authored/spec~1sources.yaml/stages", @@ -772,20 +772,20 @@ "verifier": "vintages" } ], - "configuration_field_count": 42216, - "consumed_field_count": 42216, + "configuration_field_count": 42217, + "consumed_field_count": 42217, "generation0_effect_counts": { - "legacy_behavior": 38530, + "legacy_behavior": 38531, "no_generation0_effect": 3686 }, "mode_counts": { "compiler_semantic": 27717, "front_end_validation": 354, "identity_only": 103, - "legacy_behavior": 14042 + "legacy_behavior": 14043 }, "multiple_primary_use_field_count": 0, - "pointer_inventory_sha256": "e8d32da2dbdaa015cb4eb363084c66cf509a93dde76cd473113750120dcaef80", + "pointer_inventory_sha256": "c1e2a118a0d53db0bb628b14357a2c86db2956332f5b359c9092764f9c451a95", "resolved_binding_field_count": 9780, "unused_field_count": 0 }, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "3bfce32d85079804f869686bd6bf983dbbb7e262c05a982a9d40d85ae0f69238" + "sha256": "6eb2f999ab2d7b733673f3b13402c01fe6204445b06f8b4f508dc82bbe9e1a3e" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "3bfce32d85079804f869686bd6bf983dbbb7e262c05a982a9d40d85ae0f69238" + "sha256": "6eb2f999ab2d7b733673f3b13402c01fe6204445b06f8b4f508dc82bbe9e1a3e" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "b90f0b264f9357eeda5169416babb50f3f360927a70aa0fd6ec78aa54af50427", + "sha256": "33e20427d3bddd9ee0b9b55042cc061854442c20cb6c397951642e1d3a334c50", "stages": 37 }, "failures": [], "observed": { - "sha256": "b90f0b264f9357eeda5169416babb50f3f360927a70aa0fd6ec78aa54af50427", + "sha256": "33e20427d3bddd9ee0b9b55042cc061854442c20cb6c397951642e1d3a334c50", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "e2a40490a6b705bab15bf2e7dff8eea4d7841ad70d3b7858874778d4e4829021" + "sha256": "d468a00f74ce64da5470df8487f932a8dd5db2b758f58397c6279b3c19fa9b1b" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "e2a40490a6b705bab15bf2e7dff8eea4d7841ad70d3b7858874778d4e4829021" + "sha256": "d468a00f74ce64da5470df8487f932a8dd5db2b758f58397c6279b3c19fa9b1b" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "fa02861d576da88cc771651aa309fb87bd8ac9427157b4175bced510f20d6d4d" + "sha256": "70c36826e568cc8b2e16687b8ac674618573be09b3b553dd47e5517dd4a17bf2" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "a1e98deaa1f5a451c7d0e1f380a7bf50a82b0cbc94753b885b7cf9a11e7d0005" + "spec_sha256": "b73115661e39ca2263821022aa5c5f50c453a96fb13e8af4e231428962ce0b2e" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "a1e98deaa1f5a451c7d0e1f380a7bf50a82b0cbc94753b885b7cf9a11e7d0005" + "spec_sha256": "b73115661e39ca2263821022aa5c5f50c453a96fb13e8af4e231428962ce0b2e" }, "status": "pass" } diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index 26702e36c3..feb222fb89 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -13,6 +13,48 @@ any other feed, and `test_us_chronicle_feed.py` fails if the parity resources, the generator and the pin disagree, or if the scope file changes without a new pin. +## The 2026-10-11 re-pin: BEA private group health insurance (microcosm#454) + +Decision d1317 made BEA NIPA Table 7.8 line 17 (series B4923C) the scale +anchor of the `meps_esi_premiums` stage (`docs/us-esi-employer-premiums.md`) +and asked for the feed to carry the banked fact. The pin moved from `51aa40f` +to `cf9ece2`, a commit that is `51aa40f` plus two commits (Chronicle branch +`feed/us-51aa40f-esi-b4923c`, tag `microcosm-us-feed-454-v1`), not Chronicle +main: + +- The digit-text guard match from Chronicle main's `aa3e4de3`, and nothing + else from that commit. BEA's Section 7 workbook writes its year headers as + text (`'2024'`), and at `51aa40f` no guard could match them. Only a guard + that used to fail can pass, so every package that built before builds the + same cells. +- The `bea-nipa-private-group-health-insurance` package as + PolicyEngine/chronicle#331 merged it to main, with its concept evidence + (BEA's estimates cover active and retired employees). The test file drops + main's artifact-year restamp test, whose guard (chronicle#292) postdates + this base, and the package is not added to the US poverty hard-target + coverage contract, as on main. + +The scope adds three pairs, `bea_nipa.cy.private_group_health_insurance` +for 2023, 2024 and 2025, each built at its own year (640 pairs, 66 package +runs). Built at `cf9ece2`: + +- the feed keeps all 40,178 `51aa40f` rows byte for byte (joined on + `aggregate_fact_key`) and adds exactly the three B4923C facts: $923,195M, + $977,034M and $1,027,929M, each from `Section7All_xls.xlsx` (sha256 + `de1c34e3…`); +- compiled at 2024 with the packaged CD crosswalk, aging and the Medicaid + substitutions, all 33,250 targets are identical on both feeds, name, value + and metadata. B4923C is not compiled into a target: the stage scales to it, + and whether to calibrate the column is decision d1264. The target-parity + manifest lists it as a `deferred` reviewed exclusion (82 feed families, 32 + compiled, 53 reviewed exclusions; it was a source-absent family before); +- two independent rebuilds gave the same `facts_sha256`. + +The comparison scripts and their reports are in +`experiments/us-esi-454/feed-repin/`. The release tool now refuses a feed +whose B4923C values differ from the stage's pinned anchor +(`us_esi_premiums_anchor_feed_mismatches`). + ## The 2026-10-04 re-pin: TY2023 state AGI bands (microcosm#940) The state x AGI-band targets of #940 need the TY2023 Historic Table 2 state @@ -54,20 +96,22 @@ the first labelled US export. | Field | Value | |---|---| -| Chronicle commit | `51aa40fd28e2ca81937752515e773d123fecfb71` (`c5e5bf8` plus the TY2023 state AGI package; tag `microcosm-us-feed-940-v1`) | -| Feed file | `consumer_facts_us_51aa40f.jsonl`, 40,178 rows, 169,038,757 bytes | -| `facts_sha256` | `965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac` | +| Chronicle commit | `cf9ece2e7be11158c21fdc4dd38b44d92db552f6` (`51aa40f` plus the digit-text guard and the B4923C package; tag `microcosm-us-feed-454-v1`) | +| Feed file | `consumer_facts_us_cf9ece2.jsonl`, 40,181 rows, 169,055,585 bytes | +| `facts_sha256` | `054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57` | | Consumer fact schema | `chronicle.consumer_fact.v3`, schema file sha256 `bdb51e2a…` (unchanged from the UK pin) | -| Scope | 637 (record set, period) pairs; 63 package runs over build years 2020 to 2029 | +| Scope | 640 (record set, period) pairs; 66 package runs over build years 2020 to 2029 | | Consumer artifact | none: refused at this commit, see below | -The previous pin was `c5e5bf8aa84960c1a200ee47303b19c953092d0f` -(`consumer_facts_us_c5e5bf8.jsonl`, 39,158 rows, `facts_sha256` -`b8543739…`, 586 pairs, 62 runs); the rest of this document describes its -2026-09-18 rebuild unless it says otherwise. +The previous pins were `51aa40fd28e2ca81937752515e773d123fecfb71` +(`consumer_facts_us_51aa40f.jsonl`, 40,178 rows, `facts_sha256` +`965a29ac…`, 637 pairs, 63 runs) and, before it, +`c5e5bf8aa84960c1a200ee47303b19c953092d0f` (`consumer_facts_us_c5e5bf8.jsonl`, +39,158 rows, `facts_sha256` `b8543739…`, 586 pairs, 62 runs); the rest of this +document describes the 2026-09-18 rebuild unless it says otherwise. The feed is too large for the repository. Its home on the build machine is -`~/PolicyEngine/_buildh-runtime/inputs/consumer_facts_us_51aa40f.jsonl`, +`~/PolicyEngine/_buildh-runtime/inputs/consumer_facts_us_cf9ece2.jsonl`, beside the previous pins; `tools/build_us_target_parity_manifest.py` reads it there by default. A holder of the Chronicle commit regenerates it byte for byte with the commands below. diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md index 886d34ac05..d6e38c7a5c 100644 --- a/docs/us-esi-employer-premiums.md +++ b/docs/us-esi-employer-premiums.md @@ -108,9 +108,10 @@ employee, by coverage tier: `NOEMP` and `PEIO1COW` describe different jobs. `PEIO1COW` is the class of worker of the current job. `NOEMP` is the size of the employer of the longest -job held last year. A worker who changed jobs takes the earlier employer's -size band, and a current worker with no job last year (`NOEMP` 0) takes the -State's all-sizes column. +job held last year. A worker whose longest job last year was with a different +employer from the current one takes that other employer's size band, and a +current worker with no job last year (`NOEMP` 0) takes the State's all-sizes +column. Series III for 2025 is not yet published. The 2024 government cells are aged to 2025 by the private-sector national ratio of the same tier and measure @@ -177,24 +178,32 @@ factor times the employed policyholders' shares. | Series | CY2024 | Who it covers | |---|---:|---| -| CMS NHE Table 24, "Employer Contribution to Private Health Insurance Premiums" | **$1,047.0B** | Private, federal and State and local employers. The NHEA methodology paper (2024, p. 29) says the estimates cover "active employees, continuation of health coverage (COBRA), and retirees" of private and State and local sponsors, from MEPS-IC, and federal "employers, employees and retirees", from OPM. | -| BEA NIPA Table 7.8 line 17, B4923C, "Private group health insurance" | $977.0B | Employer contributions as a supplement to wages and salaries. BEA's State Personal Income methodology (December 2025, paragraphs 3.18 and 3.19) says its MEPS source "covers both health insurance purchased by employers for their active and retired employees" and that the federal estimate covers "active and retired federal civilian employees". | +| BEA NIPA Table 7.8 line 17, B4923C, "Private group health insurance" | **$977.0B** | Employer contributions as a supplement to wages and salaries. BEA's State Personal Income methodology (December 2025, paragraphs 3.18 and 3.19) says its MEPS source "covers both health insurance purchased by employers for their active and retired employees" and that the federal estimate covers "active and retired federal civilian employees". | +| CMS NHE Table 24, "Employer Contribution to Private Health Insurance Premiums" | $1,047.0B | Private, federal and State and local employers. The NHEA methodology paper (2024, p. 29) says the estimates cover "active employees, continuation of health coverage (COBRA), and retirees" of private and State and local sponsors, from MEPS-IC, and federal "employers, employees and retirees", from OPM. | Both series measure the same concept, and both include retirees. The stage -scales to **NHE**, the series the issue chose and the pinned feed banks -(`cms_nhe.cy2024.esi_employer_contribution_premiums`). **BEA** is recorded -beside every anchor verdict and is not gated: it is 6.7% below NHE for 2024. +scales to **BEA**, as decision d1317 (2026-10-11) ruled. **NHE** is recorded +beside every anchor verdict and is not gated: it is 7.2% above BEA for 2024, +more than the anchor gate's tolerance, so a release scaled to NHE would fail +that gate. -Two facts differ from what the issue recorded in July: +The stage pins BEA's values itself (`EMPLOYER_PREMIUM_ANCHOR`, the +Section 7 workbook by URL and SHA-256). The pinned US Chronicle feed banks the +same series (record sets `bea_nipa.cy.private_group_health_insurance`, +PolicyEngine/chronicle#331; `docs/us-chronicle-feed-repin.md`), and the +release tool refuses a feed whose values differ from the stage's +(`us_esi_premiums_anchor_feed_mismatches`), so the two cannot drift apart. + +Two facts differ from what #454 recorded in July: 1. BEA revised 2024. The issue cited $1,002.9B, from the release of - 2025-09-26. The annual update published 2026-09-30 puts 2024 at $977.034B. - The stage pins the September 2026 file by SHA-256. + 2025-09-26. The annual update published 2026-09-30 puts 2024 at $977.034B + (2023 $923.195B, unchanged; 2025 $1,027.929B). 2. The issue described NHE as the broader series because it includes retirees. BEA's does too. -NHE Table 24 ends at CY2024, so the stage refuses a 2025 build year. BEA's -2025 value does not stand in for it. +The workbook column covers 2023 to 2025, so the stage refuses any other build +year. ### What the choice moves @@ -204,21 +213,21 @@ choices; the first row is what the stage builds. | Reading | Employed column | Mean per policyholder with a positive premium | |---|---:|---:| -| **NHE over every policyholder (as built)** | **$925.7B** | **$12,515** | -| NHE over every policyholder, 65-and-over non-employed at half price | $948.1B | $12,817 | -| BEA over every policyholder | $863.9B | $11,679 | -| NHE loaded onto employed policyholders only | $1,047.0B | $14,155 | +| **BEA over every policyholder (as built, d1317)** | **$863.9B** | **$11,679** | +| BEA over every policyholder, 65-and-over non-employed at half price | $884.7B | $11,961 | +| NHE over every policyholder | $925.7B | $12,515 | | BEA loaded onto employed policyholders only | $977.0B | $13,209 | +| NHE loaded onto employed policyholders only | $1,047.0B | $14,155 | -One outside check bears on it. MEPS-IC's own national rows give the -private-sector employer total for enrolled, active employees: employees times -the share in establishments that offer insurance times the share enrolled -there, times each tier's share of enrollees and its premium less average -contribution. That is $668.6B for 2024 (65.6M enrollees) and $714.4B for 2025. -The private-sector part of the as-built column is $713.2B, 6.7% above the 2024 -figure; under the BEA reading it would be $665.6B, 0.4% below. The check -covers the private sector only (77% of the column) and shares the cells' -source, so it informs the choice without settling it. +One outside check bears on it, and decided it. MEPS-IC's own national rows +give the private-sector employer total for enrolled, active employees: +employees times the share in establishments that offer insurance times the +share enrolled there, times each tier's share of enrollees and its premium +less average contribution. That is $668.6B for 2024 (65.6M enrollees) and +$714.4B for 2025. The private-sector part of the as-built column is $665.6B, +0.4% below the 2024 figure; under the NHE reading it would be $713.2B, 6.7% +above. The check covers the private sector only (77% of the column) and +shares the cells' source. ## The pre-tax employee premium @@ -266,15 +275,15 @@ pre-calibration figures at pooled ASEC weights, target year 2024. | With a positive employer premium | 74.0M | 74.1M | | Other policyholders (priced, outside the column) | 12.7M | 13.2M | | Raw MEPS-IC total, every policyholder | $1,006.4B | $1,012.1B | -| Scale factor | 1.040 | 1.034 | -| Anchor-universe total | $1,047.0B | $1,047.0B | -| **Employer premium column** | **$925.7B** | $923.9B | +| Scale factor | 0.971 | 0.965 | +| Anchor-universe total | $977.0B | $977.0B | +| **Employer premium column** | **$863.9B** | $862.2B | | Employed share of the anchor universe | 88.4% | 88.2% | -| Other policyholders' share, not in the column | $121.3B | $123.1B | -| Mean per policyholder with a positive premium | **$12,515** | $12,476 | -| Mean per employed policyholder | $11,616 | $11,579 | -| By tier: family / self plus one / self-only | $393.1B / $198.2B / $334.4B | $393.2B / $199.7B / $331.0B | -| By employer: private / State and local / federal | $713.2B / $174.3B / $38.1B | $711.2B / $174.1B / $38.6B | +| Other policyholders' share, not in the column | $113.2B | $114.9B | +| Mean per policyholder with a positive premium | **$11,679** | $11,642 | +| Mean per employed policyholder | $10,840 | $10,805 | +| By tier: family / self plus one / self-only | $366.9B / $184.9B / $312.1B | $366.9B / $186.3B / $308.9B | +| By employer: private / State and local / federal | $665.6B / $162.7B / $35.6B | $663.7B / $162.5B / $36.0B | Of the 12.7M other policyholders in the default pool, 7.1M are retired (`PEMLR` 5), 2.1M are otherwise out of the labor force (7), 0.9M are disabled @@ -289,7 +298,7 @@ the PUF-support clone, and the clone conserves both totals. | Gate | Where | Checks | |---|---|---| | `us_esi_premiums_signal_gate` | base build, before and after the clone; release tool, on the base and on the calibrated export | Column present, finite, nonnegative and non-constant; support clones agree; the weighted share of people with a positive value is plausible; no employer premium outside the column universe or where the employer pays none; every premium is one common multiple of its MEPS-IC share. | -| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of NHE Table 24 for the release period. Employed policyholders must carry 80% to 95% of it. | +| `us_esi_premiums_anchor_gate` | release tool, on the calibrated export, for both the dense and the sparse default | The column absent or zero-mass is red. The anchor-universe total, recomputed at the release's weights, must be within **5%** of BEA NIPA 7.8 line 17 for the release period. Employed policyholders must carry 80% to 95% of it. | | `us_release_input_coverage_gate` | release tool | The input is `required` with no reviewed exclusion, so an export that drops or flattens it fails. | | `assert_required_us_release_source_columns` | the L0/refit export (`l0_refit_export.export_us_l0_refit_h5`, run by `tools/export_us_l0_refit_h5.py`), unless `--allow-missing-source-columns` is passed | The column must be present and non-constant. | | reform-coverage smoke | release tool, on the written file | Neutralizing the employer premium must lower `cbo_household_market_income` by at least $500B. | @@ -328,8 +337,10 @@ stage; the failures it waives are still written to that file. `tools/build_us_meps_ic_esi_cells.py`, extracted to `us_runtime/data/meps_ic_esi_premium_cells.json`, whose own SHA-256 the stage pins and checks on load. -- NHE and BEA files: URL, byte length and SHA-256 in the stage manifest - (`us/source_stages.json`), and URL and SHA-256 in the module. The NHE +- BEA and NHE files: URL, byte length and SHA-256 in the stage manifest + (`us/source_stages.json`), and URL and SHA-256 in the module; the BEA + values also as Chronicle feed facts, checked against the module by the + release tool. The NHE digest is of the zip archive, not of the Table 24 workbook inside it. - The base summary records the stage's full summary twice (`esi_premiums_signal`, `esi_premiums_post_clone_signal`): scale factor, raw @@ -348,13 +359,16 @@ stage; the failures it waives are still written to that file. input has it null on its ACS rows. `fill_reviewed_nulls` refuses that frame instead of default-filling it to zero, so this lineage does not build until the input is assigned or transferred for every spine. -- **A calibration target.** The column is the employed part of the NHE fact, +- **A calibration target.** The column is the employed part of the BEA fact, so the fact cannot be compiled as a sum target on it. The target-parity - manifest keeps the two `cms_nhe` ESI families as `deferred` reviewed - exclusions and records BEA's series as a source-absent family. + manifest keeps the BEA family and the two `cms_nhe` ESI families as + `deferred` reviewed exclusions. Whether and how to calibrate the column is + decision d1264. - **The standalone L0/refit export.** `tools/export_us_l0_refit_h5.py` writes an H5 without running the signal or anchor gate. By default it - refuses a frame whose column is absent or constant, and nothing more: the - two ESI gates run only in the base build and in + refuses a frame whose column is absent or constant, and its generic + input-mass parity check (default tolerance 50%) compares the column's + weighted total with the base or reference, as for every input. Neither ESI + gate runs there: they run only in the base build and in `tools/build_us_fiscal_refresh_release.py`. - **The pre-tax employee premium.** See the section above. diff --git a/experiments/us-esi-454/README.md b/experiments/us-esi-454/README.md index 84072aceb0..00e80bf535 100644 --- a/experiments/us-esi-454/README.md +++ b/experiments/us-esi-454/README.md @@ -35,3 +35,16 @@ These are pre-calibration figures at pooled ASEC weights. They are evidence about the stage, not about a release: `packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py` pins the figures the docs and registers quote to these files. + +## The feed re-pin + +`feed-repin/` holds the two checks behind the 2026-10-11 re-pin of the US +Chronicle feed to `cf9ece2`, which banks BEA B4923C, the stage's anchor +(`docs/us-chronicle-feed-repin.md`): + +- `compare_feeds.py` joins the previous and new feeds on + `aggregate_fact_key` and compares row bytes; `value_diff.json` records 0 + rows missing, 0 changed and the three added B4923C facts. +- `compare_registries.py` compiles the US fiscal target registry on both + feeds the way the target-parity generator does; `registry_diff.json` + records 33,250 targets on each and no difference. diff --git a/experiments/us-esi-454/feed-repin/compare_feeds.py b/experiments/us-esi-454/feed-repin/compare_feeds.py new file mode 100644 index 0000000000..5c8facf7b7 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/compare_feeds.py @@ -0,0 +1,72 @@ +"""Compare two US Chronicle feeds row by row (microcosm#454 re-pin). + + uv run python experiments/us-esi-454/feed-repin/compare_feeds.py \ + + +Joins on ``aggregate_fact_key`` and compares each row's exact bytes. The +re-pin is accepted only if no previous row is missing or changed; the report +lists every added row's record set, period, value and source digest. +""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + + +def _rows(path: Path) -> dict[str, bytes]: + rows: dict[str, bytes] = {} + with path.open("rb") as stream: + for line in stream: + key = json.loads(line)["aggregate_fact_key"] + if key in rows: + raise SystemExit(f"{path}: repeated aggregate_fact_key {key}") + rows[key] = line + return rows + + +def _sha256(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def main() -> None: + previous_path, new_path, report_path = (Path(arg) for arg in sys.argv[1:4]) + previous, new = _rows(previous_path), _rows(new_path) + missing = sorted(key for key in previous if key not in new) + changed = sorted( + key for key in previous if key in new and previous[key] != new[key] + ) + added = [] + for key in sorted(set(new) - set(previous)): + row = json.loads(new[key]) + added.append( + { + "aggregate_fact_key": key, + "record_set_id": row["layout"]["record_set_id"], + "period": row["period"], + "value": row["value"], + "source_sha256": row["source"].get("source_sha256"), + } + ) + report = { + "previous": { + "file": previous_path.name, + "sha256": _sha256(previous_path), + "rows": len(previous), + }, + "new": {"file": new_path.name, "sha256": _sha256(new_path), "rows": len(new)}, + "previous_rows_missing": len(missing), + "previous_rows_changed": len(changed), + "rows_added": added, + } + Path(report_path).write_text(json.dumps(report, indent=1, sort_keys=True) + "\n") + print(json.dumps({k: v for k, v in report.items() if k != "rows_added"}, indent=1)) + if missing or changed: + raise SystemExit("previous rows missing or changed") + + +if __name__ == "__main__": + main() diff --git a/experiments/us-esi-454/feed-repin/compare_registries.py b/experiments/us-esi-454/feed-repin/compare_registries.py new file mode 100644 index 0000000000..b2d3228c40 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/compare_registries.py @@ -0,0 +1,73 @@ +"""Compile the US fiscal target registry on two feeds and compare (microcosm#454). + + uv run python experiments/us-esi-454/feed-repin/compare_registries.py \ + + +Uses the same compile as tools/build_us_target_parity_manifest.py (target +period 2024, age targets, canonical congressional-district crosswalk, the +reviewed Medicaid substitutions) and compares every target's name, value and +metadata. +""" + +from __future__ import annotations + +import dataclasses +import importlib.util +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] + + +def _generator(): + path = ROOT / "tools" / "build_us_target_parity_manifest.py" + spec = importlib.util.spec_from_file_location("parity_generator", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def _registry(generator, path: Path) -> dict[str, dict]: + facts, _ = generator._load_feed(path) + registry = generator.compile_us_fiscal_target_registry( + facts, + target_period=generator.TARGET_PERIOD, + congressional_district_vintage_crosswalk=( + generator.load_congressional_district_vintage_crosswalk( + generator.default_congressional_district_vintage_crosswalk_path() + ) + ), + age_targets=True, + ) + registry, _ = generator.apply_us_medicaid_enrollment_substitutions(registry) + out = {} + for spec in registry.specs: + fields = ( + dataclasses.asdict(spec) if dataclasses.is_dataclass(spec) else vars(spec) + ) + out[spec.name] = json.loads(json.dumps(fields, default=str, sort_keys=True)) + return out + + +def main() -> None: + previous_path, new_path, report_path = (Path(arg) for arg in sys.argv[1:4]) + generator = _generator() + previous = _registry(generator, previous_path) + new = _registry(generator, new_path) + report = { + "previous_targets": len(previous), + "new_targets": len(new), + "only_previous": sorted(set(previous) - set(new)), + "only_new": sorted(set(new) - set(previous)), + "changed": sorted(n for n in previous if n in new and previous[n] != new[n]), + } + Path(report_path).write_text(json.dumps(report, indent=1, sort_keys=True) + "\n") + print(json.dumps(report, indent=1)) + if report["only_previous"] or report["only_new"] or report["changed"]: + raise SystemExit("the compiled registry differs between the feeds") + + +if __name__ == "__main__": + main() diff --git a/experiments/us-esi-454/feed-repin/registry_diff.json b/experiments/us-esi-454/feed-repin/registry_diff.json new file mode 100644 index 0000000000..71a272d8b0 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/registry_diff.json @@ -0,0 +1,7 @@ +{ + "changed": [], + "new_targets": 33250, + "only_new": [], + "only_previous": [], + "previous_targets": 33250 +} diff --git a/experiments/us-esi-454/feed-repin/value_diff.json b/experiments/us-esi-454/feed-repin/value_diff.json new file mode 100644 index 0000000000..3155fd32c1 --- /dev/null +++ b/experiments/us-esi-454/feed-repin/value_diff.json @@ -0,0 +1,46 @@ +{ + "new": { + "file": "consumer_facts_us_cf9ece2.jsonl", + "rows": 40181, + "sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57" + }, + "previous": { + "file": "consumer_facts_us_51aa40f.jsonl", + "rows": 40178, + "sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac" + }, + "previous_rows_changed": 0, + "previous_rows_missing": 0, + "rows_added": [ + { + "aggregate_fact_key": "ledger.aggregate_fact.v2:45595c3331d5f2486d15838e", + "period": { + "type": "calendar_year", + "value": 2024 + }, + "record_set_id": "bea_nipa.cy2024.private_group_health_insurance", + "source_sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "value": 977034000000 + }, + { + "aggregate_fact_key": "ledger.aggregate_fact.v2:4950df5409fe51cd6d7afba6", + "period": { + "type": "calendar_year", + "value": 2023 + }, + "record_set_id": "bea_nipa.cy2023.private_group_health_insurance", + "source_sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "value": 923195000000 + }, + { + "aggregate_fact_key": "ledger.aggregate_fact.v2:c78f781d8aaf93cad7570d3f", + "period": { + "type": "calendar_year", + "value": 2025 + }, + "record_set_id": "bea_nipa.cy2025.private_group_health_insurance", + "source_sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "value": 1027929000000 + } + ] +} diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json index a7b9d0ac53..b0e2bdb2c5 100644 --- a/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json +++ b/experiments/us-esi-454/receipts/stage_on_pool_2022_2024.json @@ -2,23 +2,11 @@ "anchor_gate": { "details": { "anchor": { - "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", - "row": "Employer Contribution to Private Health Insurance Premiums", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", - "values": { - "2023": 975700000000, - "2024": 1047000000000 - } - }, - "anchor_universe_employer_total": 1047000000000.0, - "anchor_value": 1047000000000.0, - "cross_check": { - "label": "Private group health insurance", + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", "line": 17, "published": "2026-09-30", - "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "row": "line 17, series B4923C, Private group health insurance", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", @@ -29,29 +17,42 @@ "2025": 1027929000000 } }, - "cross_check_ratio": 1.0716106092520834, + "anchor_universe_employer_total": 977034000000.0, + "anchor_value": 977034000000.0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "cross_check_ratio": 0.9331747851002865, "employed_share_band": [ 0.8, 0.95 ], "employed_share_of_anchor_universe": 0.8824300567320681, "employer_premium_by_sector": { - "federal": 38621718711.29335, - "private": 711190949595.6248, - "state_and_local": 174091601091.55725 + "federal": 36040814058.6149, + "private": 663665461554.1658, + "state_and_local": 162457892436.37872 }, - "employer_premium_mean_per_positive_person": 12475.548691763059, + "employer_premium_mean_per_positive_person": 11641.867469444154, "employer_premium_positive_persons": 74057205.19599111, - "employer_premium_total": 923904269398.4752, - "other_policyholder_employer_total": 123095730601.5247, + "employer_premium_total": 862164168049.1594, + "other_policyholder_employer_total": 114869831950.84059, "private_active_cross_check": { - "employer_premium_private_sector": 711190949595.6248, + "employer_premium_private_sector": 663665461554.1658, "meps_ic_private_active_employer_total": 668573745863.8779, - "ratio": 1.0637434598582394 + "ratio": 0.9926585745550477 }, "relative_error": 0.0, "relative_tolerance": 0.05, - "scale_factor": 1.034460355190118, + "scale_factor": 0.9653323196493044, "time_period": 2024, "weighted_employed_policyholders": 79790597.17620382, "weighted_other_policyholders": 13156453.970184255, @@ -63,20 +64,20 @@ "by_vintage": { "2022": { "employer_premium_positive_persons": 24684189.26891587, - "employer_premium_total": 310331531290.6173, - "mean_per_positive_person": 12572.077126365637, + "employer_premium_total": 289593560021.9647, + "mean_per_positive_person": 11731.945370660484, "weighted_persons": 108650789.68666667 }, "2023": { "employer_premium_positive_persons": 24692526.63040858, - "employer_premium_total": 307546511529.20874, - "mean_per_positive_person": 12455.044237978813, + "employer_premium_total": 286994649804.61224, + "mean_per_positive_person": 11622.733230190439, "weighted_persons": 108650789.68666664 }, "2024": { "employer_premium_positive_persons": 24680489.296666663, - "employer_premium_total": 306026226578.64923, - "mean_per_positive_person": 12399.520240467073, + "employer_premium_total": 285575958222.5826, + "mean_per_positive_person": 11570.919635744516, "weighted_persons": 108650789.68666667 } }, @@ -109,24 +110,24 @@ "issue": "PolicyEngine/microcosm#454", "other_policyholder_sensitivity": { "as_built_other_priced_as_active": { - "employer_premium_total": 923904269398.4752, - "mean_per_positive_person": 12475.548691763057 - }, - "bea_anchor_all_on_employed": { - "employer_premium_total": 977034000000.0001, - "mean_per_positive_person": 13192.963431637698 - }, - "bea_anchor_other_priced_as_active": { "employer_premium_total": 862164168049.1594, "mean_per_positive_person": 11641.867469444152 }, + "nhe_anchor_all_on_employed": { + "employer_premium_total": 1047000000000.0, + "mean_per_positive_person": 14137.71958081773 + }, + "nhe_anchor_other_priced_as_active": { + "employer_premium_total": 923904269398.4752, + "mean_per_positive_person": 12475.548691763057 + }, "other_65_plus_priced_at_half": { - "employer_premium_total": 946325497990.1058, - "mean_per_positive_person": 12778.304224223311 + "employer_premium_total": 883087093221.8387, + "mean_per_positive_person": 11924.391298385674 }, "other_not_priced_all_anchor_on_employed": { - "employer_premium_total": 1047000000000.0, - "mean_per_positive_person": 14137.71958081773 + "employer_premium_total": 977034000000.0001, + "mean_per_positive_person": 13192.963431637698 }, "raw_other_65_plus_share_of_other": 0.4030440846499319, "weighted_other_policyholders_65_plus": 5861452.143127043, @@ -533,8 +534,8 @@ ], "puf_support_clone": { "anchor_gate": { - "anchor_universe_employer_total": 1047000000000.0002, - "employer_premium_total": 923904269398.4753, + "anchor_universe_employer_total": 977034000000.0001, + "employer_premium_total": 862164168049.1594, "failures": [], "passed": true }, @@ -544,7 +545,7 @@ "passed": true } }, - "seconds": 35.4, + "seconds": 37.5, "seed": 0, "signal_gate": { "failures": [], @@ -552,24 +553,11 @@ }, "summary": { "anchor": { - "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", - "row": "Employer Contribution to Private Health Insurance Premiums", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", - "values": { - "2023": 975700000000, - "2024": 1047000000000 - } - }, - "anchor_universe_employer_total": 1047000000000.0, - "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", - "clone_disagreement_source_persons": 0, - "cross_check": { - "label": "Private group health insurance", + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", "line": 17, "published": "2026-09-30", - "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "row": "line 17, series B4923C, Private group health insurance", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", @@ -580,22 +568,36 @@ "2025": 1027929000000 } }, + "anchor_universe_employer_total": 977034000000.0, + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "clone_disagreement_source_persons": 0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, "employed_share_of_anchor_universe": 0.8824300567320681, "employer_premium_by_sector": { - "federal": 38621718711.29335, - "private": 711190949595.6248, - "state_and_local": 174091601091.55725 + "federal": 36040814058.6149, + "private": 663665461554.1658, + "state_and_local": 162457892436.37872 }, "employer_premium_by_tier": { - "employee_plus_one": 199685222296.09024, - "family": 393209175914.58295, - "single": 331009871187.80206 + "employee_plus_one": 186341214403.85696, + "family": 366932888233.55176, + "single": 308890065411.75073 }, - "employer_premium_mean_per_positive_person": 12475.548691763059, + "employer_premium_mean_per_positive_person": 11641.867469444154, "employer_premium_outside_universe_rows": 0, "employer_premium_positive_persons": 74057205.19599111, "employer_premium_positive_share": 0.2272025370134953, - "employer_premium_total": 923904269398.4752, + "employer_premium_total": 862164168049.1594, "employer_premium_where_employer_pays_none_rows": 0, "employer_premium_without_raw_share_rows": 0, "meps_ic_private_active_employer_total": { @@ -604,7 +606,7 @@ }, "negative_rows": 0, "nonfinite_rows": 0, - "other_policyholder_employer_total": 123095730601.5247, + "other_policyholder_employer_total": 114869831950.84059, "raw_anchor_universe_total": 1012121919169.708, "raw_employer_share_band": [ 6000.0, @@ -613,8 +615,8 @@ "raw_employer_share_mean_per_positive_person": 12059.958246993665, "raw_employer_share_total": 893126802552.6951, "raw_over_published_employer_mean": 1.030960994318845, - "scale_factor": 1.034460355190118, - "scale_factor_spread": 0.0, + "scale_factor": 0.9653323196493044, + "scale_factor_spread": 2.220446049250313e-16, "source_columns_missing": [], "weighted_employed_policyholders": 79790597.17620382, "weighted_employed_policyholders_by_payment": { diff --git a/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json index 61760666d6..f10ee8482d 100644 --- a/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json +++ b/experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json @@ -2,23 +2,11 @@ "anchor_gate": { "details": { "anchor": { - "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", - "row": "Employer Contribution to Private Health Insurance Premiums", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", - "values": { - "2023": 975700000000, - "2024": 1047000000000 - } - }, - "anchor_universe_employer_total": 1046999999999.9998, - "anchor_value": 1047000000000.0, - "cross_check": { - "label": "Private group health insurance", + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", "line": 17, "published": "2026-09-30", - "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "row": "line 17, series B4923C, Private group health insurance", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", @@ -29,29 +17,42 @@ "2025": 1027929000000 } }, - "cross_check_ratio": 1.0716106092520832, + "anchor_universe_employer_total": 977034000000.0, + "anchor_value": 977034000000.0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, + "cross_check_ratio": 0.9331747851002865, "employed_share_band": [ 0.8, 0.95 ], "employed_share_of_anchor_universe": 0.8841568448185655, "employer_premium_by_sector": { - "federal": 38124900985.39438, - "private": 713245726890.8082, - "state_and_local": 174341588648.83527 + "federal": 35577196284.01511, + "private": 665582927915.0277, + "state_and_local": 162691174521.41943 }, - "employer_premium_mean_per_positive_person": 12514.898608320464, + "employer_premium_mean_per_positive_person": 11678.587819371325, "employer_premium_positive_persons": 73968814.73011559, - "employer_premium_total": 925712216525.0378, - "other_policyholder_employer_total": 121287783474.96194, + "employer_premium_total": 863851298720.4623, + "other_policyholder_employer_total": 113182701279.53773, "private_active_cross_check": { - "employer_premium_private_sector": 713245726890.8082, + "employer_premium_private_sector": 665582927915.0277, "meps_ic_private_active_employer_total": 668573745863.8779, - "ratio": 1.0668168340490378 + "ratio": 0.995526569855079 }, - "relative_error": -2.220446049250313e-16, + "relative_error": 0.0, "relative_tolerance": 0.05, - "scale_factor": 1.0403508274129025, + "scale_factor": 0.9708291597999409, "time_period": 2024, "weighted_employed_policyholders": 79689927.57027404, "weighted_other_policyholders": 12732394.803675905, @@ -63,20 +64,20 @@ "by_vintage": { "2023": { "employer_premium_positive_persons": 24692526.63040858, - "employer_premium_total": 309297757165.9197, - "mean_per_positive_person": 12525.966329627154, + "employer_premium_total": 288628868075.3078, + "mean_per_positive_person": 11688.915937823243, "weighted_persons": 108650789.68666664 }, "2024": { "employer_premium_positive_persons": 24680489.296666663, - "employer_premium_total": 307768815338.1516, - "mean_per_positive_person": 12470.126164788748, + "employer_premium_total": 287202098113.7495, + "mean_per_positive_person": 11636.807304000205, "weighted_persons": 108650789.68666667 }, "2025": { "employer_premium_positive_persons": 24595798.803040348, - "employer_premium_total": 308645644020.96655, - "mean_per_positive_person": 12548.713969103293, + "employer_premium_total": 288020332531.40497, + "mean_per_positive_person": 11710.143461402931, "weighted_persons": 108650789.68666668 } }, @@ -109,24 +110,24 @@ "issue": "PolicyEngine/microcosm#454", "other_policyholder_sensitivity": { "as_built_other_priced_as_active": { - "employer_premium_total": 925712216525.038, - "mean_per_positive_person": 12514.898608320465 - }, - "bea_anchor_all_on_employed": { - "employer_premium_total": 977034000000.0, - "mean_per_positive_person": 13208.728618470228 - }, - "bea_anchor_other_priced_as_active": { "employer_premium_total": 863851298720.4623, "mean_per_positive_person": 11678.587819371325 }, + "nhe_anchor_all_on_employed": { + "employer_premium_total": 1047000000000.0, + "mean_per_positive_person": 14154.61372228431 + }, + "nhe_anchor_other_priced_as_active": { + "employer_premium_total": 925712216525.038, + "mean_per_positive_person": 12514.898608320465 + }, "other_65_plus_priced_at_half": { - "employer_premium_total": 948058754883.9099, - "mean_per_positive_person": 12817.006171357754 + "employer_premium_total": 884704524851.2379, + "mean_per_positive_person": 11960.50697958582 }, "other_not_priced_all_anchor_on_employed": { - "employer_premium_total": 1047000000000.0, - "mean_per_positive_person": 14154.61372228431 + "employer_premium_total": 977034000000.0, + "mean_per_positive_person": 13208.728618470228 }, "raw_other_65_plus_share_of_other": 0.4069439742577271, "weighted_other_policyholders_65_plus": 5730053.275654608, @@ -534,8 +535,8 @@ ], "puf_support_clone": { "anchor_gate": { - "anchor_universe_employer_total": 1046999999999.9998, - "employer_premium_total": 925712216525.0378, + "anchor_universe_employer_total": 977033999999.9999, + "employer_premium_total": 863851298720.4622, "failures": [], "passed": true }, @@ -545,7 +546,7 @@ "passed": true } }, - "seconds": 35.1, + "seconds": 40.8, "seed": 0, "signal_gate": { "failures": [], @@ -553,24 +554,11 @@ }, "summary": { "anchor": { - "coverage": "active employees, COBRA enrollees and retirees of private, federal and State and local employers", - "row": "Employer Contribution to Private Health Insurance Premiums", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", - "values": { - "2023": 975700000000, - "2024": 1047000000000 - } - }, - "anchor_universe_employer_total": 1046999999999.9998, - "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", - "clone_disagreement_source_persons": 0, - "cross_check": { - "label": "Private group health insurance", + "coverage": "employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", "line": 17, "published": "2026-09-30", - "relation": "second estimate of the same concept: employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19)", + "row": "line 17, series B4923C, Private group health insurance", "series": "B4923C", "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", @@ -581,22 +569,36 @@ "2025": 1027929000000 } }, + "anchor_universe_employer_total": 977034000000.0, + "cells_sha256": "29a5502abbd921e70007d6bcec8ed1ae065b309cacfa80f49f9bc4dd9a28ec93", + "clone_disagreement_source_persons": 0, + "cross_check": { + "relation": "second estimate of the same concept: employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper, 2024, p. 29)", + "row": "Employer Contribution to Private Health Insurance Premiums", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "source": "https://www.cms.gov/files/zip/nhe-tables.zip", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "values": { + "2023": 975700000000, + "2024": 1047000000000 + } + }, "employed_share_of_anchor_universe": 0.8841568448185655, "employer_premium_by_sector": { - "federal": 38124900985.39438, - "private": 713245726890.8082, - "state_and_local": 174341588648.83527 + "federal": 35577196284.01511, + "private": 665582927915.0277, + "state_and_local": 162691174521.41943 }, "employer_premium_by_tier": { - "employee_plus_one": 198151264745.7921, - "family": 393140740531.96185, - "single": 334420211247.28394 + "employee_plus_one": 184909763896.50455, + "family": 366869026060.08105, + "single": 312072508763.87665 }, - "employer_premium_mean_per_positive_person": 12514.898608320464, + "employer_premium_mean_per_positive_person": 11678.587819371325, "employer_premium_outside_universe_rows": 0, "employer_premium_positive_persons": 73968814.73011559, "employer_premium_positive_share": 0.2269313609943412, - "employer_premium_total": 925712216525.0378, + "employer_premium_total": 863851298720.4623, "employer_premium_where_employer_pays_none_rows": 0, "employer_premium_without_raw_share_rows": 0, "meps_ic_private_active_employer_total": { @@ -605,7 +607,7 @@ }, "negative_rows": 0, "nonfinite_rows": 0, - "other_policyholder_employer_total": 121287783474.96194, + "other_policyholder_employer_total": 113182701279.53773, "raw_anchor_universe_total": 1006391279183.8038, "raw_employer_share_band": [ 6000.0, @@ -614,8 +616,8 @@ "raw_employer_share_mean_per_positive_person": 12029.498394730888, "raw_employer_share_total": 889807738056.072, "raw_over_published_employer_mean": 1.0307192789751904, - "scale_factor": 1.0403508274129025, - "scale_factor_spread": 0.0, + "scale_factor": 0.9708291597999409, + "scale_factor_spread": 2.220446049250313e-16, "source_columns_missing": [], "weighted_employed_policyholders": 79689927.57027404, "weighted_employed_policyholders_by_payment": { diff --git a/experiments/us-esi-454/run_stage_on_pinned_pool.py b/experiments/us-esi-454/run_stage_on_pinned_pool.py index ce95fa8f5c..80c80ed051 100644 --- a/experiments/us-esi-454/run_stage_on_pinned_pool.py +++ b/experiments/us-esi-454/run_stage_on_pinned_pool.py @@ -87,7 +87,7 @@ def _other_policyholder_sensitivity(frame, target_year: int) -> dict[str, object outside the column priced at half the active cell, and nobody outside the column priced at all (the whole anchor loaded onto employed workers). It also restates the as-built column and the all-on-employed reading against - BEA NIPA 7.8 line 17, the second estimate of the anchor's concept. + CMS NHE Table 24, the second estimate of the anchor's concept. """ person = esi._person_with_state(frame) @@ -98,7 +98,7 @@ def _other_policyholder_sensitivity(frame, target_year: int) -> dict[str, object age = person["A_AGE"].to_numpy(dtype=np.float64) older = other & (age >= 65) anchor = float(esi.EMPLOYER_PREMIUM_ANCHOR["values"][str(target_year)]) - bea = float(esi.EMPLOYER_PREMIUM_CROSS_CHECK["values"][str(target_year)]) + nhe = float(esi.EMPLOYER_PREMIUM_CROSS_CHECK["values"][str(target_year)]) employed = float(weights[universe] @ raw[universe]) everyone = float(weights @ raw) older_raw = float(weights[older] @ raw[older]) @@ -116,8 +116,8 @@ def column(total_raw: float, total_anchor: float = anchor) -> dict[str, float]: "as_built_other_priced_as_active": column(everyone), "other_65_plus_priced_at_half": column(everyone - 0.5 * older_raw), "other_not_priced_all_anchor_on_employed": column(employed), - "bea_anchor_other_priced_as_active": column(everyone, bea), - "bea_anchor_all_on_employed": column(employed, bea), + "nhe_anchor_other_priced_as_active": column(everyone, nhe), + "nhe_anchor_all_on_employed": column(employed, nhe), "weighted_other_policyholders_by_pemlr": { str(code): float(weights[other & (pemlr == code)].sum()) for code in sorted(set(pemlr[other].tolist())) diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py index ba54255837..f86db2dfb8 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/field_usage.py @@ -26,9 +26,9 @@ ) from .schemas import load_schema_registry -EXPECTED_AUTHORED_FIELD_COUNT = 32_436 +EXPECTED_AUTHORED_FIELD_COUNT = 32_437 EXPECTED_RESOLVED_BINDING_FIELD_COUNT = 9_780 -EXPECTED_CONFIGURATION_FIELD_COUNT = 42_216 +EXPECTED_CONFIGURATION_FIELD_COUNT = 42_217 class FieldUsageError(AssertionError): @@ -485,8 +485,8 @@ def _path_inventory(rows: Sequence[tuple[str, object]]) -> tuple[int, str]: "d6782c5de5bbed1bdc6bf653c4a6d4aadcad4ccc72d35e1092e130fcb04680a3", ), "source_stages": ( - 1_748, - "dcc891c12eb675ee7f21c903995db98c3b8c6ccba569d71f0fd59b589abce04c", + 1_749, + "d0979722ea5153bd152e5e1c1201f5d530d11b21aecad18aca38473962caa062", ), "spine_assembly_household_mass_shares": ( 2, diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index f82307a13f..91192b4b93 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,18 +350,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "e2a40490a6b705bab15bf2e7dff8eea4d7841ad70d3b7858874778d4e4829021", + "full_checkpoint": "d468a00f74ce64da5470df8487f932a8dd5db2b758f58397c6279b3c19fa9b1b", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "3bfce32d85079804f869686bd6bf983dbbb7e262c05a982a9d40d85ae0f69238", + "late_resource_semantics": "6eb2f999ab2d7b733673f3b13402c01fe6204445b06f8b4f508dc82bbe9e1a3e", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", "seed_map": "5641418e4a815da24744d59dfcf82fb11aa2f2ddc290ffcab9627383ea6fa682", "seed_protocol": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb", - "source_manifest": "b90f0b264f9357eeda5169416babb50f3f360927a70aa0fd6ec78aa54af50427", + "source_manifest": "33e20427d3bddd9ee0b9b55042cc061854442c20cb6c397951642e1d3a334c50", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } diff --git a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json index 5b46e935e1..0aa420db00 100644 --- a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json +++ b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json @@ -3,15 +3,15 @@ "country": "us", "scope": "us_fiscal_targets", "source_repo": "PolicyEngine/chronicle", - "source_commit": "51aa40fd28e2ca81937752515e773d123fecfb71", - "build": "tools/build_us_chronicle_feed.py --skip-artifact: one `chronicle build-bundle --year Y --source ` per build year in chronicle_feed_scope.json, rows kept for exactly the scope's (record set, period) pairs and sorted by aggregate_fact_key. The commit is c5e5bf8 plus only the TY2023 Historic Table 2 state AGI-band package (tag microcosm-us-feed-940-v1, chronicle#291). Bare feed: `chronicle build-consumer-artifact` refuses it at this commit; see docs/us-chronicle-feed-repin.md.", + "source_commit": "cf9ece2e7be11158c21fdc4dd38b44d92db552f6", + "build": "tools/build_us_chronicle_feed.py --skip-artifact: one `chronicle build-bundle --year Y --source ` per build year in chronicle_feed_scope.json, rows kept for exactly the scope's (record set, period) pairs and sorted by aggregate_fact_key. The commit is c5e5bf8 plus the TY2023 Historic Table 2 state AGI-band package (51aa40f, tag microcosm-us-feed-940-v1, chronicle#291), the digit-text guard match from chronicle main's aa3e4de3, and the BEA NIPA Table 7.8 line 17 (B4923C) package as chronicle#331 merged it (branch feed/us-51aa40f-esi-b4923c, tag microcosm-us-feed-454-v1, microcosm#454). Bare feed: `chronicle build-consumer-artifact` refuses it at this commit; see docs/us-chronicle-feed-repin.md.", "artifact_schema_version": null, "consumer_fact_schema_versions": [ "chronicle.consumer_fact.v3" ], "consumer_fact_schema_sha256": "bdb51e2a8115634633ba7448c4005930fd9c0bfbade5e1b079b6bc24da485d3d", - "fact_row_count": 40178, - "facts_sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac", + "fact_row_count": 40181, + "facts_sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57", "manifest_sha256": null, - "scope_sha256": "f090102099b3d2c799a8f975a8156101dbb18925b526c3791a785f68944d020d" + "scope_sha256": "f4a218fc3be534f3e447f25bf1ec4064d9f3db6b72a55874ed5db4823c922239" } diff --git a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json index d3647ec2b6..596954435f 100644 --- a/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json +++ b/packages/microcosm-build/src/microcosm/build/us/chronicle_feed_scope.json @@ -2,9 +2,9 @@ "version": 1, "country": "us", "source_repo": "PolicyEngine/chronicle", - "source_commit": "51aa40fd28e2ca81937752515e773d123fecfb71", - "rule": "The (record_set_id, period) pairs of the pinned feed consumer_facts_buildn_v9_4.jsonl, plus the 51 TY2023 Historic Table 2 state AGI-band record sets (irs_soi.ty2023.historic_table_2.state_agi., microcosm#940); each built from the named package with `chronicle build-bundle --year --source `.", - "pair_count": 637, + "source_commit": "cf9ece2e7be11158c21fdc4dd38b44d92db552f6", + "rule": "The (record_set_id, period) pairs of the pinned feed consumer_facts_buildn_v9_4.jsonl, plus the 51 TY2023 Historic Table 2 state AGI-band record sets (irs_soi.ty2023.historic_table_2.state_agi., microcosm#940), plus the three BEA NIPA Table 7.8 line 17 (B4923C) record sets (bea_nipa.cy.private_group_health_insurance, 2023 to 2025, microcosm#454); each built from the named package with `chronicle build-bundle --year --source `.", + "pair_count": 640, "runs": { "2020": [ "packages/bea/nipa_total_wages_salaries", @@ -28,6 +28,7 @@ "packages/bea/nipa_pension_contributions", "packages/bea/nipa_personal_income_components", "packages/bea/nipa_personal_income_disposition", + "packages/bea/nipa_private_group_health_insurance", "packages/bea/nipa_total_wages_salaries", "packages/bea/regional_personal_income_state", "packages/cbo/individual_income_tax_receipts_2026_02", @@ -54,6 +55,7 @@ "packages/irs_soi/w2_statistics_2020" ], "2024": [ + "packages/bea/nipa_private_group_health_insurance", "packages/bea/nipa_total_wages_salaries", "packages/cbo/revenue_projections_income_by_source_2026_02", "packages/census/pep_2024_national_age_sex", @@ -74,6 +76,7 @@ "packages/usda_snap/fy69_to_current" ], "2025": [ + "packages/bea/nipa_private_group_health_insurance", "packages/cbo/revenue_projections_income_by_source_2026_02" ], "2026": [ @@ -259,6 +262,14 @@ "package": "packages/bea/nipa_personal_income_disposition", "build_year": 2023 }, + { + "record_set_id": "bea_nipa.cy2023.private_group_health_insurance", + "period_type": "calendar_year", + "period_value": "2023", + "package_id": "bea-nipa-private-group-health-insurance", + "package": "packages/bea/nipa_private_group_health_insurance", + "build_year": 2023 + }, { "record_set_id": "bea_nipa.cy2023.proprietors_income", "period_type": "calendar_year", @@ -315,6 +326,14 @@ "package": "packages/bea/nipa_personal_income_components", "build_year": 2023 }, + { + "record_set_id": "bea_nipa.cy2024.private_group_health_insurance", + "period_type": "calendar_year", + "period_value": "2024", + "package_id": "bea-nipa-private-group-health-insurance", + "package": "packages/bea/nipa_private_group_health_insurance", + "build_year": 2024 + }, { "record_set_id": "bea_nipa.cy2024.total_wages_salaries", "period_type": "calendar_year", @@ -323,6 +342,14 @@ "package": "packages/bea/nipa_total_wages_salaries", "build_year": 2024 }, + { + "record_set_id": "bea_nipa.cy2025.private_group_health_insurance", + "period_type": "calendar_year", + "period_value": "2025", + "package_id": "bea-nipa-private-group-health-insurance", + "package": "packages/bea/nipa_private_group_health_insurance", + "build_year": 2025 + }, { "record_set_id": "bea_regional.cy2023.state_contributions_for_government_social_insurance", "period_type": "calendar_year", diff --git a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json index 0d55025c85..6555a7a3e2 100644 --- a/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/release_input_coverage_manifest.json @@ -72,7 +72,7 @@ "status": "required" }, "employer_sponsored_insurance_premiums": { - "note": "Employer share of the premium for employed current ESI policyholders (PolicyEngine/microcosm#454), written by the meps_esi_premiums base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person files) and MEPS-IC tier x firm-size x State cells, scaled over every policyholder to CMS NHE Table 24. A release must persist it: CBO household market income (gov.household.cbo_market_income_additions) and every ESI-to-wages reform bind through it. Required with NO reviewed exclusion; absent on pre-#454 bases by construction — the intended red gate until a base built with the stage ships. The release also holds the anchor-universe total to the anchor (us_esi_premiums_anchor_gate).", + "note": "Employer share of the premium for employed current ESI policyholders (PolicyEngine/microcosm#454), written by the meps_esi_premiums base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person files) and MEPS-IC tier x firm-size x State cells, scaled over every policyholder to BEA NIPA 7.8 line 17 (B4923C). A release must persist it: CBO household market income (gov.household.cbo_market_income_additions) and every ESI-to-wages reform bind through it. Required with NO reviewed exclusion; absent on pre-#454 bases by construction — the intended red gate until a base built with the stage ships. The release also holds the anchor-universe total to the anchor (us_esi_premiums_anchor_gate).", "status": "required" }, "employment_income_before_lsr": { @@ -580,7 +580,7 @@ "neutralized_variable": "employer_sponsored_insurance_premiums", "parameter_changes": {}, "period": 2024, - "reason": "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums to CBO household market income through gov.household.cbo_market_income_additions, its only consumer, so neutralizing it lowers cbo_household_market_income by the weighted employer-premium total: about $926 billion at 2024 (the employed policyholders' part of CMS NHE Table 24's $1,047.0 billion, the meps_esi_premiums anchor; the stage measured $925.7 billion over 74.0 million weighted employed policyholders on the default 2023-2025 pool, before calibration). The floor sits at about half of that. A structural zero means the column was dropped or the stage never ran." + "reason": "PolicyEngine-US 2.2.1 adds employer_sponsored_insurance_premiums to CBO household market income through gov.household.cbo_market_income_additions, its only consumer, so neutralizing it lowers cbo_household_market_income by the weighted employer-premium total: about $864 billion at 2024 (the employed policyholders' part of BEA NIPA 7.8 line 17's $977.0 billion, the meps_esi_premiums anchor; the stage measured $863.9 billion over 74.0 million weighted employed policyholders on the default 2023-2025 pool, before calibration). The floor sits at about half of that. A structural zero means the column was dropped or the stage never ran." }, { "binding_inputs": [ diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 202943187a..fe90d9ab9f 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1793,16 +1793,6 @@ "size_bytes": 1765717, "publication": "MEPS-IC 2024 Series III (State and local government, by government type and size and census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released)." }, - { - "kind": "administrative_table", - "format": "xlsx", - "vintage": "NHE 2024 release (data through CY2024)", - "locator": "https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "size_bytes": 520391, - "measure": "NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024", - "publication": "scale anchor: employer contributions for active employees, COBRA enrollees and retirees of private, federal and State and local employers" - }, { "kind": "administrative_table", "format": "xlsx", @@ -1811,7 +1801,17 @@ "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "size_bytes": 944273, "measure": "NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025", - "publication": "cross-check, not gated: a second estimate of employer contributions for active and retired employees, recorded beside the anchor verdict" + "publication": "scale anchor (decision d1317): employer contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); the pinned US Chronicle feed banks the same values as bea_nipa.cy.private_group_health_insurance" + }, + { + "kind": "administrative_table", + "format": "xlsx", + "vintage": "NHE 2024 release (data through CY2024)", + "locator": "https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx", + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "size_bytes": 520391, + "measure": "NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024", + "publication": "cross-check, not gated: a second estimate of employer contributions for active employees, COBRA enrollees and retirees of private, federal and State and local employers, recorded beside the anchor verdict" } ], "operations": [ @@ -1837,13 +1837,14 @@ "government_cell_aging": "MEPS-IC 2024 Series III x private-sector national 2025/2024 ratio of the same tier and measure", "scaling": "one factor, anchor / weighted raw share of the anchor universe; written to the universe only", "anchor": { - "source": "https://www.cms.gov/files/zip/nhe-tables.zip", - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", - "row": "Employer Contribution to Private Health Insurance Premiums", + "source": "https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx", + "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", + "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "row": "line 17, series B4923C, Private group health insurance", "values": { - "2023": 975700000000, - "2024": 1047000000000 + "2023": 923195000000, + "2024": 977034000000, + "2025": 1027929000000 } } } @@ -1854,7 +1855,7 @@ "nonnegative_outputs": [ "employer_sponsored_insurance_premiums" ], - "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active employees, COBRA enrollees and retirees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals CMS NHE Table 24's employer contribution for the build year ($1,047.0B CY2024, banked as cms_nhe.cy2024.esi_employer_contribution_premiums), the anchor microcosm#454 chose; the column carries the employed policyholders' part. BEA NIPA 7.8 line 17 ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of NHE Table 24; employed share within its band)." + "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active employees, COBRA enrollees and retirees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), the anchor decision d1317 chose; the column carries the employed policyholders' part. CMS NHE Table 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees, which the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band)." }, { "stage": "prior_year_income", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index b9b0500e0e..262556a741 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe + sha256: 50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24 stage_manifest: version: 1 country: us @@ -1829,15 +1829,6 @@ stages: publication: 'MEPS-IC 2024 Series III (State and local government, by government type and size and census division): Tables III.C/D/E.1 and III.C/D/E.2. 2024 is the latest published year for this series (the 2025 series is not yet released).' - - kind: administrative_table - format: xlsx - vintage: NHE 2024 release (data through CY2024) - locator: https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx - sha256: a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420 - size_bytes: 520391 - measure: NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024 - publication: 'scale anchor: employer contributions for active employees, COBRA enrollees and retirees - of private, federal and State and local employers' - kind: administrative_table format: xlsx vintage: BEA annual update published 2026-09-30 @@ -1845,8 +1836,19 @@ stages: sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd size_bytes: 944273 measure: NIPA Table 7.8 line 17 Private group health insurance (B4923C), CY2023-2025 - publication: 'cross-check, not gated: a second estimate of employer contributions for active and retired - employees, recorded beside the anchor verdict' + publication: 'scale anchor (decision d1317): employer contributions for active and retired employees + (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); the pinned US Chronicle + feed banks the same values as bea_nipa.cy.private_group_health_insurance' + - kind: administrative_table + format: xlsx + vintage: NHE 2024 release (data through CY2024) + locator: https://www.cms.gov/files/zip/nhe-tables.zip!Table 24 Employer-Sponsored Private Health Insurance.xlsx + sha256: a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420 + size_bytes: 520391 + measure: NHE Table 24 Employer Contribution to Private Health Insurance Premiums, CY2023-2024 + publication: 'cross-check, not gated: a second estimate of employer contributions for active employees, + COBRA enrollees and retirees of private, federal and State and local employers, recorded beside + the anchor verdict' operations: - kind: read_table table: person @@ -1870,13 +1872,14 @@ stages: tier and measure scaling: one factor, anchor / weighted raw share of the anchor universe; written to the universe only anchor: - source: https://www.cms.gov/files/zip/nhe-tables.zip - sha256: a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420 - table: NHE Table 24 Employer-Sponsored Private Health Insurance - row: Employer Contribution to Private Health Insurance Premiums + source: https://apps.bea.gov/national/Release/XLS/Survey/Section7All_xls.xlsx + sha256: de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd + table: NIPA Table 7.8 Supplements to Wages and Salaries by Type + row: line 17, series B4923C, Private group health insurance values: - '2023': 975700000000 - '2024': 1047000000000 + '2023': 923195000000 + '2024': 977034000000 + '2025': 1027929000000 outputs: - employer_sponsored_insurance_premiums nonnegative_outputs: @@ -1902,18 +1905,18 @@ stages: United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder''s share so - the weighted anchor-universe total equals CMS NHE Table 24''s employer contribution for the build - year ($1,047.0B CY2024, banked as cms_nhe.cy2024.esi_employer_contribution_premiums), the anchor microcosm#454 - chose; the column carries the employed policyholders'' part. BEA NIPA 7.8 line 17 ($977.0B CY2024 - in the 2026-09-30 vintage, revised from $1,002.9B) is a second estimate of the same active-plus-retired - concept and is recorded beside the verdict, as is MEPS-IC''s own private-sector enrollment total for - active employees. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced - as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The - stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint - from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release - gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the - raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent - of NHE Table 24; employed share within its band).' + the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B + CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), + the anchor decision d1317 chose; the column carries the employed policyholders'' part. CMS NHE Table + 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded + beside the verdict, as is MEPS-IC''s own private-sector enrollment total for active employees, which + the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, + so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities + are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US + treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), + so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, + cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe + total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band).' - stage: prior_year_income survey: CPS ASEC (prior year) source: https://www.census.gov/programs-surveys/cps.html diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json index 23e9607ad3..870197d07e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_feed_families.json @@ -1,6 +1,6 @@ { - "feed": "consumer_facts_us_51aa40f.jsonl", - "feed_sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac", + "feed": "consumer_facts_us_cf9ece2.jsonl", + "feed_sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57", "families": { "bea_nipa.business_current_transfer_receipts": 1, "bea_nipa.defined_contribution_actual_contributions": 1, @@ -22,6 +22,7 @@ "bea_nipa.personal_outlays": 1, "bea_nipa.personal_saving": 1, "bea_nipa.personal_saving_rate": 1, + "bea_nipa.private_group_health_insurance": 3, "bea_nipa.proprietors_income": 1, "bea_nipa.rental_income_of_persons": 1, "bea_nipa.social_security_benefits": 1, diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index d983578f37..24ca56329e 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -1,8 +1,8 @@ { "schema_version": 1, "reference": { - "feed": "consumer_facts_us_51aa40f.jsonl", - "feed_sha256": "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac", + "feed": "consumer_facts_us_cf9ece2.jsonl", + "feed_sha256": "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57", "target_period": "2024", "registry_compile": "compile_us_fiscal_target_registry(age_targets=True, canonical CD vintage crosswalk) + apply_us_medicaid_enrollment_substitutions; one national + state + congressional-district surface for every artifact (microcosm#449)", "us_data_source": "retired us-data pipeline (archived): utils/loss.py (eCPS loss matrix), db/etl_national_targets.py, db/etl_*.py, utils/national_target_parity.py", @@ -234,13 +234,13 @@ }, "bea_nipa.private_group_health_insurance": { "status": "reviewed_exclusion", - "classification": "source_absent", - "reason": "BEA NIPA Table 7.8 line 17 private group health insurance (B4923C), the employer-contribution aggregate the retired pipeline calibrated employer_sponsored_insurance_premiums to (loss.py 1_002.9e9). The pinned feed carries no fact for it.", - "evidence": "retired us-data pipeline (archived 2e044571) loss.py line 33; feed bea_nipa families carry no group-health series", + "classification": "deferred", + "reason": "BEA NIPA Table 7.8 line 17 (B4923C), employer contributions for private group health insurance: the scale anchor of the meps_esi_premiums base stage (microcosm#454, decision d1317). The stage prices every current ESI policyholder from MEPS-IC, scales that universe to this fact (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19: contributions for active and retired employees) and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this fact, and the release tool refuses a feed whose values differ from the stage's pinned anchor (us_esi_premiums_anchor_feed_mismatches).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $977.0B at scale factor 0.971; employed column $863.9B (88.4%); other policyholders $113.2B; private-sector part of the column $665.6B against MEPS-IC's own private active-employee total of $668.6B (2024)", "fence": { - "origin": "us-data loss.py employer_sponsored_insurance_premiums target", - "purpose": "anchor employer-paid ESI premiums. The series counts contributions for active and retired employees (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19), the same concept as NHE Table 24.", - "verdict_basis": "source-absent from the feed: the meps_esi_premiums stage pins the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., published 2026-09-30: $977.034B CY2024, revised from $1,002.9B) and records the anchor-universe total against it beside the release anchor verdict; the stage scales to NHE Table 24. Compile it here once the ledger banks the series." + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "purpose": "anchor the CBO market-income ESI component once the column has a live producer. The BEA annual update of 2026-09-30 revised CY2024 from the $1,002.9B the retired target used to $977.034B.", + "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. Whether and how to calibrate the column is decision d1264." } }, "bea_nipa.proprietors_income": { @@ -460,23 +460,23 @@ "cms_nhe.esi_employer_contribution_premiums": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums base stage (microcosm#454) scales to this fact over the fact's own universe, every current ESI policyholder (NHEA methodology paper 2024, p. 29: active employees, COBRA and retirees), and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this all-employer fact.", - "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $1,047.0B at scale factor 1.040; employed column $925.7B (88.4%); other policyholders $121.3B; private-sector part of the column $713.2B against MEPS-IC's own private active-employee total of $668.6B (2024)", + "reason": "NHE Table 24 employer contribution to ESI premiums, a second estimate of the concept the meps_esi_premiums stage scales to (microcosm#454): employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper 2024, p. 29). Decision d1317 chose BEA NIPA 7.8 line 17 as the stage's anchor instead; this fact is 7.2% above it for CY2024. It is neither compiled nor gated: us_esi_premiums_anchor_gate records the anchor-universe total against it beside the verdict (cross_check_ratio).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: anchor-universe total $977.0B, 0.933 of NHE's $1,047.0B; scaled to NHE instead, the employed column would be $925.7B", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", - "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NHE Table 24 and NIPA 7.8 line 17 both count employer contributions for active and retired employees (NHEA methodology paper; BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in CY2024 and is recorded beside the gate verdict.", - "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. A calibration target needs an exported anchor-universe quantity or a banked active-employee fact." + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "purpose": "a cross-check on the ESI anchor's level: an independent estimate of the same active-plus-retired concept from the national health accounts.", + "verdict_basis": "deferred, not dropped: the stage's anchor is BEA (d1317), and this fact, like it, includes retirees, so it is not a sum target on the employed column. It is recorded beside the anchor verdict. Whether and how to calibrate the column is decision d1264." } }, "cms_nhe.esi_private_employer_contribution_premiums": { "status": "reviewed_exclusion", "classification": "deferred", - "reason": "NHE Table 24 private-employer contribution to ESI premiums, the private part of the all-employer fact the meps_esi_premiums stage scales to (microcosm#454). It is neither compiled nor gated: the stage scales one factor to the all-employer total over every current ESI policyholder, and nothing holds the private-sector part of the column to this fact, which also counts private retirees and COBRA enrollees. The private part is instead recorded beside the anchor verdict against MEPS-IC's own active-employee total (us_esi_premiums_anchor_gate, private_active_cross_check).", - "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: private-sector part of the column $713.2B; MEPS-IC private active-employee total $668.6B (2024)", + "reason": "NHE Table 24 private-employer contribution to ESI premiums, the private part of the NHE all-employer fact (microcosm#454). It is neither compiled nor gated: the stage scales one factor to BEA NIPA 7.8 line 17 over every current ESI policyholder, and nothing holds the private-sector part of the column to this fact, which also counts private retirees and COBRA enrollees. The private part is instead recorded beside the anchor verdict against MEPS-IC's own active-employee total (us_esi_premiums_anchor_gate, private_active_cross_check).", + "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: private-sector part of the column $665.6B; MEPS-IC private active-employee total $668.6B (2024)", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581). The v9.2 feed banks the NHE series for the #454 gate; the NIPA series the us-data target used is not in the feed (source-absent family bea_nipa.private_group_health_insurance).", - "purpose": "anchor the CBO market-income ESI component once the column has a live producer. NHE Table 24 and NIPA 7.8 line 17 both count employer contributions for active and retired employees (NHEA methodology paper; BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in CY2024 and is recorded beside the gate verdict.", - "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. A calibration target needs an exported anchor-universe quantity or a banked active-employee fact." + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "purpose": "a sector cross-check that private employers' share of the ESI total is in range.", + "verdict_basis": "deferred, not dropped: no gate reads this fact. The private part of the column is compared with MEPS-IC's active-employee total instead, because this fact includes private retirees and COBRA enrollees. Whether and how to calibrate the column is decision d1264." } }, "cms_nhe.medicaid_title_xix_expenditures": { diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index c2085f5546..50b954a756 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -273,6 +273,7 @@ US_ESI_PREMIUMS_STAGE_NAME, derive_us_employer_esi_premiums_from_manifest, load_meps_ic_esi_premium_cells, + us_esi_premiums_anchor_feed_mismatches, us_esi_premiums_anchor_gate, us_esi_premiums_signal_gate, us_esi_premiums_stage_spec, @@ -1372,6 +1373,7 @@ "US_ESI_PREMIUMS_STAGE_NAME", "derive_us_employer_esi_premiums_from_manifest", "load_meps_ic_esi_premium_cells", + "us_esi_premiums_anchor_feed_mismatches", "us_esi_premiums_anchor_gate", "us_esi_premiums_signal_gate", "us_esi_premiums_stage_spec", diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py index 10b17df110..86f612526c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/esi_premiums.py @@ -58,17 +58,22 @@ national small-firm cell times the State's total-to-national ratio. One factor scales every policyholder's raw share so the weighted total equals +BEA NIPA Table 7.8 line 17 (series B4923C, "Private group health insurance") +for the build year, the anchor decision d1317 chose. BEA's State Personal +Income methodology (December 2025, paragraphs 3.18-3.19) says its MEPS source +"covers both health insurance purchased by employers for their active and +retired employees" and its federal estimate covers "active and retired federal +civilian employees". The pinned US Chronicle feed carries the same values +(record sets ``bea_nipa.cy.private_group_health_insurance``), and the +release tool refuses a feed whose values differ from the stage's. + CMS NHE Table 24's employer contribution to private health insurance premiums -for the build year, the anchor microcosm#454 chose. Its methodology paper says -the estimate covers "active employees, continuation of health coverage -(COBRA), and retirees" of private and State and local sponsors (from MEPS-IC) -and federal "employers, employees and retirees" (from OPM). BEA NIPA Table 7.8 -line 17 (series B4923C) is a second estimate of the same concept, 6.7% lower -for 2024: BEA's State Personal Income methodology (December 2025, paragraphs -3.18-3.19) says its MEPS source "covers both health insurance purchased by -employers for their active and retired employees" and its federal estimate -covers "active and retired federal civilian employees". It is recorded as a -cross-check, not gated. +is a second estimate of the same concept, 7.2% higher for 2024 (its +methodology paper: "active employees, continuation of health coverage +(COBRA), and retirees"). It is recorded beside the anchor verdict, as is +MEPS-IC's own private-sector active-employee total, which the private part of +the column matches within half a percent on the pinned pools. Neither is +gated. Not produced here ----------------- @@ -84,7 +89,7 @@ import hashlib import json -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass from importlib.resources import files from types import MappingProxyType @@ -118,6 +123,7 @@ "load_meps_ic_esi_premium_cells", "meps_ic_private_active_employer_totals", "refuse_unassigned_us_esi_premiums", + "us_esi_premiums_anchor_feed_mismatches", "us_esi_premiums_anchor_gate", "us_esi_premiums_signal_gate", "us_esi_premiums_stage_spec", @@ -197,50 +203,53 @@ ) _NHE_TABLES_URL = "https://www.cms.gov/files/zip/nhe-tables.zip" -#: CMS NHE Table 24 (2024 National Health Expenditure Accounts, data through -#: CY2024), row "Employer Contribution to Private Health Insurance Premiums": -#: private, federal and State and local employers, for active employees, COBRA -#: enrollees and retirees. The anchor microcosm#454 chose; banked in the -#: pinned feed as cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums. -#: ``sha256`` is the digest of the zip archive at ``source`` (520,391 bytes), -#: not of the Table 24 workbook inside it. -EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( - { - "source": _NHE_TABLES_URL, - "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", - "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", - "row": "Employer Contribution to Private Health Insurance Premiums", - "coverage": ( - "active employees, COBRA enrollees and retirees of private, " - "federal and State and local employers" - ), - # Integer dollars: the manifest round-trips through YAML. - "values": {"2023": 975_700_000_000, "2024": 1_047_000_000_000}, - } -) #: BEA NIPA Table 7.8 line 17 (B4923C), millions of dollars converted to #: dollars, from the annual update published 2026-09-30 (file created -#: 2026-09-28). That update revised 2024 from the $1,002.9B the retired -#: us-data loss matrix pinned. A second estimate of the same concept; recorded, -#: not gated. -EMPLOYER_PREMIUM_CROSS_CHECK: Mapping[str, Any] = MappingProxyType( +#: 2026-09-28; that update revised 2024 from the $1,002.9B the retired us-data +#: loss matrix pinned). The anchor decision d1317 chose. The pinned US +#: Chronicle feed banks the same three values (chronicle#331). +EMPLOYER_PREMIUM_ANCHOR: Mapping[str, Any] = MappingProxyType( { "source": _BEA_SECTION7_URL, "sha256": "de1c34e37da9b8b8d765efc0611cc653102373fe522fe9218c7689067b29b7fd", "table": "NIPA Table 7.8 Supplements to Wages and Salaries by Type", + "row": "line 17, series B4923C, Private group health insurance", "line": 17, "series": "B4923C", - "label": "Private group health insurance", "published": "2026-09-30", + "feed_record_set": "bea_nipa.cy{year}.private_group_health_insurance", + "coverage": ( + "employer contributions for active and retired employees (BEA " + "State Personal Income methodology, December 2025, paragraphs " + "3.18-3.19)" + ), + # Integer dollars: the manifest round-trips through YAML. "values": { "2023": 923_195_000_000, "2024": 977_034_000_000, "2025": 1_027_929_000_000, }, + } +) +#: CMS NHE Table 24 (2024 National Health Expenditure Accounts, data through +#: CY2024), row "Employer Contribution to Private Health Insurance Premiums": +#: private, federal and State and local employers, for active employees, COBRA +#: enrollees and retirees; banked in the pinned feed as +#: cms_nhe.cy{2023,2024}.esi_employer_contribution_premiums. A second estimate +#: of the same concept; recorded, not gated. ``sha256`` is the digest of the +#: zip archive at ``source`` (520,391 bytes), not of the Table 24 workbook +#: inside it. +EMPLOYER_PREMIUM_CROSS_CHECK: Mapping[str, Any] = MappingProxyType( + { + "source": _NHE_TABLES_URL, + "sha256": "a09ef6d3e84e25d745047a47b6b08a0d96b303085b4c725b67ce67a0eb0c4420", + "table": "NHE Table 24 Employer-Sponsored Private Health Insurance", + "row": "Employer Contribution to Private Health Insurance Premiums", + "values": {"2023": 975_700_000_000, "2024": 1_047_000_000_000}, "relation": ( "second estimate of the same concept: employer contributions for " - "active and retired employees (BEA State Personal Income " - "methodology, December 2025, paragraphs 3.18-3.19)" + "active employees, COBRA enrollees and retirees (NHEA methodology " + "paper, 2024, p. 29)" ), } ) @@ -653,11 +662,51 @@ def meps_ic_private_active_employer_totals( return totals +def us_esi_premiums_anchor_feed_mismatches( + facts: Iterable[Mapping[str, Any]], +) -> list[str]: + """Where the Chronicle feed's B4923C facts differ from the stage's anchor. + + The stage pins the BEA values it scales to; the pinned US feed banks the + same series (``bea_nipa.cy.private_group_health_insurance``). The + release tool calls this on the facts it compiles targets from, so the + producer's anchor and the banked fact cannot drift apart unseen. Returns + one message per anchor year whose fact is missing, repeated or different. + """ + + template = str(EMPLOYER_PREMIUM_ANCHOR["feed_record_set"]) + wanted = { + template.format(year=year): (year, int(value)) + for year, value in EMPLOYER_PREMIUM_ANCHOR["values"].items() + } + found: dict[str, list[Any]] = {record_set: [] for record_set in wanted} + for fact in facts: + layout = fact.get("layout") + record_set = ( + layout.get("record_set_id") if isinstance(layout, Mapping) else None + ) + if record_set in found: + found[record_set].append(fact.get("value")) + problems: list[str] = [] + for record_set, (year, expected) in sorted(wanted.items()): + values = found[record_set] + if not values: + problems.append(f"{record_set}: no fact in the feed ({year}).") + elif len(values) > 1: + problems.append(f"{record_set}: {len(values)} facts in the feed ({year}).") + elif values[0] != expected: + problems.append( + f"{record_set}: feed value {values[0]!r} is not the stage's " + f"anchor {expected} ({year})." + ) + return problems + + def _anchor(year: int) -> float: values = EMPLOYER_PREMIUM_ANCHOR["values"] if str(year) not in values: raise SourceRuntimeError( - f"US ESI premium stage has no NHE Table 24 anchor for {year}; pinned " + f"US ESI premium stage has no BEA NIPA 7.8 line 17 anchor for {year}; pinned " f"years: {sorted(values)}." ) return float(values[str(year)]) @@ -689,7 +738,7 @@ def derive_us_employer_esi_premiums_from_manifest( operation: SourceOperationSpec, context: SourceRuntimeContext, ) -> pd.DataFrame: - """Assign MEPS-IC employer shares and scale them to the NHE anchor. + """Assign MEPS-IC employer shares and scale them to the BEA anchor. The factor is set over every policyholder, the anchor's universe; the column carries the employed policyholders' part. @@ -1088,17 +1137,17 @@ def us_esi_premiums_signal_gate(frame: Frame) -> GateResult: def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult: """Hold a release's weighted ESI employer premium total to its anchor. - Runs on the calibrated export (dense and sparse). The anchor, CMS NHE - Table 24, counts employer contributions for every policyholder, so the + Runs on the calibrated export (dense and sparse). The anchor, BEA NIPA + 7.8 line 17, counts employer contributions for every policyholder, so the gate compares the anchor-universe total: the column (employed policyholders) plus the same scale factor applied to the other policyholders' MEPS-IC shares, recomputed from the raw ASEC columns at the frame's weights. It fails when the column is absent or zero-mass, when the raw columns are absent, when that total leaves ±``ANCHOR_RELATIVE_TOLERANCE`` of the anchor for ``time_period``, or when - the employed share of it leaves its band. BEA NIPA 7.8 line 17 and - MEPS-IC's own private-sector enrollment total are recorded beside the - verdict, not gated. + the employed share of it leaves its band. CMS NHE Table 24 and MEPS-IC's + own private-sector enrollment total are recorded beside the verdict, not + gated. """ person = frame.table("person") @@ -1143,7 +1192,7 @@ def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult failures.append(f"{US_ESI_EMPLOYER_PREMIUM_COLUMN}: zero weighted mass.") anchor = EMPLOYER_PREMIUM_ANCHOR["values"].get(year) if anchor is None: - failures.append(f"no NHE Table 24 employer-contribution anchor for {year}.") + failures.append(f"no BEA NIPA 7.8 line 17 anchor for {year}.") if summary["source_columns_missing"]: failures.append( _missing_source_failure( @@ -1178,8 +1227,8 @@ def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult if abs(relative) > ANCHOR_RELATIVE_TOLERANCE: failures.append( "employer premiums over every policyholder total " - f"${universe_total / 1e9:,.1f}B, {relative:+.1%} from NHE " - f"Table 24 ${float(anchor) / 1e9:,.1f}B ({year}); tolerance " + f"${universe_total / 1e9:,.1f}B, {relative:+.1%} from BEA NIPA 7.8 " + f"line 17 ${float(anchor) / 1e9:,.1f}B ({year}); tolerance " f"±{ANCHOR_RELATIVE_TOLERANCE:.0%}." ) low, high = _EMPLOYED_SHARE_BAND @@ -1189,9 +1238,9 @@ def us_esi_premiums_anchor_gate(frame: Frame, *, time_period: int) -> GateResult f"employed policyholders carry {employed_share:.4f} of the " f"anchor-universe employer total, outside [{low}, {high}]." ) - bea = EMPLOYER_PREMIUM_CROSS_CHECK["values"].get(year) - if bea is not None and np.isfinite(universe_total): - details["cross_check_ratio"] = universe_total / float(bea) + nhe = EMPLOYER_PREMIUM_CROSS_CHECK["values"].get(year) + if nhe is not None and np.isfinite(universe_total): + details["cross_check_ratio"] = universe_total / float(nhe) active = dict(summary["meps_ic_private_active_employer_total"]) if year in active: private = float(dict(summary["employer_premium_by_sector"])["private"]) diff --git a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py index 5dd6be077c..3a0dd0e3d9 100644 --- a/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py +++ b/packages/microcosm-build/tests/engine_contract/us/test_spec_engine_coverage_tool.py @@ -82,21 +82,21 @@ def test_us_coverage_is_exact_complete_and_honest( assert_coverage_complete(coverage_report) assert coverage_report["status"] == "pass" fields = coverage_report["field_usage"] - assert fields["configuration_field_count"] == 42_216 - assert fields["authored_normative_field_count"] == 32_436 + assert fields["configuration_field_count"] == 42_217 + assert fields["authored_normative_field_count"] == 32_437 assert fields["resolved_binding_field_count"] == 9_780 - assert fields["consumed_field_count"] == 42_216 + assert fields["consumed_field_count"] == 42_217 assert fields["unused_field_count"] == 0 assert fields["multiple_primary_use_field_count"] == 0 assert fields["claim_count"] == 49 assert fields["mode_counts"] == { - "legacy_behavior": 14_042, + "legacy_behavior": 14_043, "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert fields["generation0_effect_counts"] == { - "legacy_behavior": 38_530, + "legacy_behavior": 38_531, "no_generation0_effect": 3_686, } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py index 4babaf7512..79d903c887 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_engine_field_usage.py @@ -95,22 +95,22 @@ def _mutated_bundle( def test_exact_complete_ledger_has_one_primary_mode_per_pointer(field_ledger) -> None: - assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_216 + assert len(field_ledger.fields) == EXPECTED_CONFIGURATION_FIELD_COUNT == 42_217 assert field_ledger.source_counts == { - "authored": 32_436, + "authored": 32_437, "resolved_bindings": 9_780, } assert field_ledger.mode_counts == { - "legacy_behavior": 14_042, + "legacy_behavior": 14_043, "compiler_semantic": 27_717, "front_end_validation": 354, "identity_only": 103, } assert field_ledger.generation0_effect_counts == { - "legacy_behavior": 38_530, + "legacy_behavior": 38_531, "no_generation0_effect": 3_686, } - assert len({field.pointer for field in field_ledger.fields}) == 42_216 + assert len({field.pointer for field in field_ledger.fields}) == 42_217 def test_eligibility_concepts_are_validation_not_generation0_behavior( diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 4d4d52c0b4..33df6af7b8 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe", + "sha256": "50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py index fb05929633..3a572c7107 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_chronicle_feed.py @@ -31,8 +31,8 @@ _TEST_PATHS = paths_for("microcosm-build") _ROOT = _TEST_PATHS.repository -_CHRONICLE_COMMIT = "51aa40fd28e2ca81937752515e773d123fecfb71" -_FACTS_SHA256 = "965a29ac9458edb58a05184917ee4a19772fd3dc7b4588db6eb6da5803641aac" +_CHRONICLE_COMMIT = "cf9ece2e7be11158c21fdc4dd38b44d92db552f6" +_FACTS_SHA256 = "054e5684005a7b143dcb84eabac634135ce9b2b33bb57b540602e474a4ba7c57" _SCHEMA_SHA256 = "bdb51e2a8115634633ba7448c4005930fd9c0bfbade5e1b079b6bc24da485d3d" @@ -55,7 +55,7 @@ def test_pin_records_the_rebuilt_bare_feed() -> None: assert pin.source_repo == "PolicyEngine/chronicle" assert pin.source_commit == _CHRONICLE_COMMIT assert pin.scope == "us_fiscal_targets" - assert pin.fact_row_count == 40178 + assert pin.fact_row_count == 40181 assert pin.facts_sha256 == _FACTS_SHA256 assert pin.consumer_fact_schema_versions == ("chronicle.consumer_fact.v3",) assert pin.consumer_fact_schema_sha256 == _SCHEMA_SHA256 @@ -137,8 +137,9 @@ def test_scope_names_every_pinned_pair_once() -> None: scope = builder.load_scope(scope_path) raw = scope["raw"] assert scope["commit"] == load_us_chronicle_feed().source_commit - assert len(scope["pairs"]) == 637 == raw["pair_count"] - assert sum(len(p) for p in scope["runs"].values()) == 63 + assert len(scope["pairs"]) == 640 == raw["pair_count"] + # 63 package runs, plus the B4923C package in its three years (#454). + assert sum(len(p) for p in scope["runs"].values()) == 66 assert set(scope["runs"]) == { 2020, 2021, diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py index fb06908cfc..b45f5e764a 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py @@ -4,7 +4,7 @@ * **Conservation**: the weighted employer share over every policyholder (the column plus the other policyholders' share at the same scale factor) equals - the CMS NHE Table 24 anchor of the build year, whatever the frame. The + the BEA NIPA 7.8 line 17 anchor of the build year, whatever the frame. The column total is the anchor times the employed policyholders' share of the weighted raw mass, so it equals the anchor exactly when no one outside the column carries an employer share. @@ -74,10 +74,11 @@ from test_support.paths import paths_for TIME_PERIOD = 2024 -#: CMS NHE Table 24 employer contribution, CY2024: the scale anchor. -ANCHOR = 1_047_000e6 -#: BEA NIPA Table 7.8 line 17, CY2024 (2026-09-30 vintage): the cross-check. -BEA = 977_034e6 +#: BEA NIPA Table 7.8 line 17 (B4923C), CY2024, 2026-09-30 vintage: the scale +#: anchor (decision d1317). +ANCHOR = 977_034e6 +#: CMS NHE Table 24 employer contribution, CY2024: the recorded cross-check. +NHE = 1_047_000e6 EMPLOYER = US_ESI_EMPLOYER_PREMIUM_COLUMN _REPOSITORY_ROOT = paths_for("microcosm-build").repository _CELLS = load_meps_ic_esi_premium_cells() @@ -240,8 +241,9 @@ def test_manifest_pins_the_cells_the_anchor_and_every_source_pdf(self) -> None: spec = us_esi_premiums_stage_spec() employer = spec.operations[1].parameters assert employer["cells_sha256"] == esi._CELLS_SHA256 - assert employer["anchor"]["table"].startswith("NHE Table 24") + assert employer["anchor"]["table"].startswith("NIPA Table 7.8") assert employer["anchor"]["row"] == esi.EMPLOYER_PREMIUM_ANCHOR["row"] + assert "B4923C" in employer["anchor"]["row"] assert employer["anchor"]["values"]["2024"] == ANCHOR assert employer["anchor_universe"] == "NOW_OWNGRP == 1" assert "PEMLR in (1, 2)" in employer["universe"] @@ -640,7 +642,7 @@ def test_state_may_already_sit_on_the_person_table(self) -> None: _values(on_household, EMPLOYER), _values(on_person, EMPLOYER) ) - @pytest.mark.parametrize("year", [2023, 2024]) + @pytest.mark.parametrize("year", [2023, 2024, 2025]) def test_each_pinned_year_scales_to_its_own_anchor(self, year) -> None: result = _run(_realistic_rows(60), time_period=year) total = us_esi_premiums_summary(result)["anchor_universe_employer_total"] @@ -702,11 +704,11 @@ def test_a_frame_with_no_employer_paid_mass_cannot_be_scaled(self, other) -> Non with pytest.raises(SourceRuntimeError, match="no weighted employer-paid"): _run([{"NOW_HIPAID": 3}, other]) - @pytest.mark.parametrize("year", [2025, 2030]) + @pytest.mark.parametrize("year", [2022, 2026, 2030]) def test_an_unpinned_build_year_is_refused(self, year) -> None: - # NHE Table 24 ends at CY2024; BEA's 2025 value does not stand in. + # The pinned BEA workbook column covers 2023 to 2025. with pytest.raises( - SourceRuntimeError, match=f"no NHE Table 24 anchor for {year}" + SourceRuntimeError, match=f"no BEA NIPA 7.8 line 17 anchor for {year}" ): _run([{}], time_period=year) @@ -802,8 +804,8 @@ def test_passes_on_a_staged_frame_and_reports_the_lineage(self) -> None: ) assert details["scale_factor_spread"] <= 1e-9 * details["scale_factor"] assert details["cells_sha256"] == esi._CELLS_SHA256 - assert details["anchor"]["row"].startswith("Employer Contribution") - assert details["cross_check"]["series"] == "B4923C" + assert details["anchor"]["series"] == "B4923C" + assert details["cross_check"]["row"].startswith("Employer Contribution") assert sum(details["employer_premium_by_tier"].values()) == pytest.approx( column ) @@ -975,7 +977,7 @@ def test_fails_outside_the_stated_tolerance(self, scale) -> None: self._export(weight_scale=scale), time_period=TIME_PERIOD ) assert not gate.passed - assert any("from NHE Table 24" in f for f in gate.failures) + assert any("from BEA NIPA 7.8 line 17" in f for f in gate.failures) def test_the_verdict_is_over_every_policyholder_not_the_column(self) -> None: gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) @@ -1005,15 +1007,16 @@ def test_reweighting_workers_against_other_policyholders_is_red(self) -> None: _column_total(frame) ) assert not gate.passed - assert any("from NHE Table 24" in f for f in gate.failures) + assert any("from BEA NIPA 7.8 line 17" in f for f in gate.failures) assert any("employed policyholders carry" in f for f in gate.failures) def test_the_cross_checks_are_recorded_not_gated(self) -> None: gate = us_esi_premiums_anchor_gate(self._export(), time_period=TIME_PERIOD) assert gate.passed, gate.failures - # BEA's estimate of the same concept is 6.7% below NHE's for 2024. - assert gate.details["cross_check_ratio"] == pytest.approx(ANCHOR / BEA) - assert ANCHOR / BEA - 1 > esi.ANCHOR_RELATIVE_TOLERANCE + # NHE's estimate of the same concept is 7.2% above BEA's for 2024, so + # a release scaled to NHE would fail this gate. + assert gate.details["cross_check_ratio"] == pytest.approx(ANCHOR / NHE) + assert NHE / ANCHOR - 1 > esi.ANCHOR_RELATIVE_TOLERANCE active = gate.details["private_active_cross_check"] assert active["meps_ic_private_active_employer_total"] == pytest.approx( 668.6e9, rel=5e-4 @@ -1066,13 +1069,12 @@ def test_a_frame_without_the_household_state_fails_both_gates(self) -> None: for f in gate.failures ) - @pytest.mark.parametrize("year", [2025, 2031]) + @pytest.mark.parametrize("year", [2022, 2031]) def test_a_year_without_an_anchor_is_red(self, year) -> None: gate = us_esi_premiums_anchor_gate(self._export(), time_period=year) assert not gate.passed assert any( - f"no NHE Table 24 employer-contribution anchor for {year}" in f - for f in gate.failures + f"no BEA NIPA 7.8 line 17 anchor for {year}" in f for f in gate.failures ) def test_details_serialize_for_the_release_evidence_file(self) -> None: @@ -1080,7 +1082,7 @@ def test_details_serialize_for_the_release_evidence_file(self) -> None: payload = json.loads(json.dumps(dict(gate.details))) assert payload["anchor"]["sha256"] == esi.EMPLOYER_PREMIUM_ANCHOR["sha256"] assert payload["anchor"]["values"]["2024"] == ANCHOR - assert payload["cross_check"]["values"]["2024"] == BEA + assert payload["cross_check"]["values"]["2024"] == NHE class TestOperatorBoundary: @@ -1098,6 +1100,41 @@ def test_a_raw_source_frame_that_carries_the_outputs_is_refused(self) -> None: assert_operator_free_source_frame(_run(_realistic_rows(40)), label="raw") +class TestAnchorFeedCheck: + @staticmethod + def _facts(values: dict[int, object]) -> list[dict]: + return [ + { + "layout": { + "record_set_id": f"bea_nipa.cy{year}.private_group_health_insurance" + }, + "value": value, + } + for year, value in values.items() + ] + [{"layout": {"record_set_id": "bea_nipa.cy2024.total_wages_salaries"}}] + + def test_the_banked_series_matches_the_anchor(self) -> None: + values = { + int(year): value + for year, value in esi.EMPLOYER_PREMIUM_ANCHOR["values"].items() + } + assert values == { + 2023: 923_195_000_000, + 2024: 977_034_000_000, + 2025: 1_027_929_000_000, + } + assert esi.us_esi_premiums_anchor_feed_mismatches(self._facts(values)) == [] + + def test_a_missing_repeated_or_different_fact_is_named(self) -> None: + facts = self._facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + facts += self._facts({2023: 923_195_000_000})[:1] + problems = esi.us_esi_premiums_anchor_feed_mismatches(facts) + assert len(problems) == 3 + assert any("cy2023" in p and "2 facts" in p for p in problems) + assert any("cy2024" in p and "1002920000000" in p for p in problems) + assert any("cy2025" in p and "no fact" in p for p in problems) + + class TestUnassignedRefusal: def test_a_staged_frame_and_a_frame_without_the_columns_pass(self) -> None: refuse_unassigned_us_esi_premiums(_run(_realistic_rows(60)), consumer="t") @@ -1437,35 +1474,50 @@ def test_a_shipped_reform_probe_binds_through_the_input(self) -> None: assert 0 < employer.min_abs_effect < ANCHOR assert "pre_tax_health_insurance_premium_neutralization" not in probes - def test_the_nhe_target_family_is_gated_not_compiled(self) -> None: + def test_the_anchor_and_cross_check_families_are_gated_not_compiled( + self, + ) -> None: manifest = json.loads( files("microcosm.build.us") .joinpath("target_parity_manifest.json") .read_text() ) families = manifest["families"] - for family in ( - "cms_nhe.esi_employer_contribution_premiums", - "cms_nhe.esi_private_employer_contribution_premiums", - ): - entry = families[family] - assert entry["status"] == "reviewed_exclusion" - assert entry["classification"] == "deferred" - assert "every current ESI policyholder" in entry["reason"] - assert "retirees" in entry["reason"] - assert "us_esi_premiums_anchor_gate" in entry["reason"] - absent = families["bea_nipa.private_group_health_insurance"] - assert absent["classification"] == "source_absent" - assert "B4923C" in absent["reason"] - assert "active and retired" in absent["fence"]["purpose"] + anchor = families["bea_nipa.private_group_health_insurance"] + assert anchor["status"] == "reviewed_exclusion" + assert anchor["classification"] == "deferred" + assert "scale anchor" in anchor["reason"] + assert "B4923C" in anchor["reason"] + assert "active and retired" in anchor["reason"] + assert "us_esi_premiums_anchor_gate" in anchor["reason"] + assert "us_esi_premiums_anchor_feed_mismatches" in anchor["reason"] + assert "d1264" in anchor["fence"]["verdict_basis"] + nhe = families["cms_nhe.esi_employer_contribution_premiums"] + assert nhe["classification"] == "deferred" + assert "retirees" in nhe["reason"] and "d1317" in nhe["reason"] + private = families["cms_nhe.esi_private_employer_contribution_premiums"] + assert private["classification"] == "deferred" + assert "no gate reads this fact" in private["fence"]["verdict_basis"] + # Each family states its own verdict basis. + assert ( + len({entry["fence"]["verdict_basis"] for entry in (anchor, nhe, private)}) + == 3 + ) + # The BEA facts are in the pinned feed now, so the family is no longer + # source-absent. + assert not any( + entry.get("classification") == "source_absent" + and family.startswith("bea_nipa.private_group_health") + for family, entry in families.items() + ) @pytest.mark.parametrize( ("pool", "scale", "holders", "column", "mean", "share"), [ # The default pool (docs/us-asec-source-pins.md) and the historical # Build J/N/P pool; figures quoted in the docs and the registers. - ("2023_2025", 1.0404, 73.97e6, 925.7e9, 12_515, 0.8842), - ("2022_2024", 1.0345, 74.06e6, 923.9e9, 12_476, 0.8824), + ("2023_2025", 0.9708, 73.97e6, 863.85e9, 11_679, 0.8842), + ("2022_2024", 0.9653, 74.06e6, 862.16e9, 11_642, 0.8824), ], ) def test_the_receipts_reproduce_the_documented_measurements( @@ -1502,12 +1554,13 @@ def test_the_receipts_reproduce_the_documented_measurements( mean, abs=1 ) details = receipt["anchor_gate"]["details"] - assert details["cross_check_ratio"] == pytest.approx(ANCHOR / BEA) + assert details["cross_check_ratio"] == pytest.approx(ANCHOR / NHE) active = details["private_active_cross_check"] assert active["meps_ic_private_active_employer_total"] == pytest.approx( 668.6e9, rel=5e-4 ) - assert 1.06 < active["ratio"] < 1.07 + # The private part of the column against MEPS-IC's own active total. + assert 0.99 < active["ratio"] < 1.0 sensitivity = receipt["other_policyholder_sensitivity"] assert sensitivity["as_built_other_priced_as_active"][ "employer_premium_total" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 657d5d7936..5584840cf8 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -7079,10 +7079,14 @@ def fake_written_stored_input_check(path, pre_export): ), ) # The invented feed is not the committed pin; this test is about the - # corridor contract, not the pin, which has its own tests. + # corridor contract, not the pin, which has its own tests. Nor does it + # carry the ESI anchor's B4923C facts (checked in test_us_esi_premiums). monkeypatch.setattr( builder, "_check_committed_us_ledger_feed_pin", lambda *a, **k: None ) + monkeypatch.setattr( + builder, "us_esi_premiums_anchor_feed_mismatches", lambda facts: [] + ) monkeypatch.setattr( builder, "compile_us_fiscal_target_registry", @@ -16169,6 +16173,19 @@ def test_main_checks_the_committed_feed_pin_before_compiling_targets() -> None: assert source.count("_check_committed_us_ledger_feed_pin(") == 1 +def test_main_holds_the_esi_anchor_to_the_feed_before_compiling_targets() -> None: + """microcosm#454: the anchor-versus-feed check runs on the pinned feed.""" + builder = _load_builder_module() + source = inspect.getsource(builder._main) + checked = source.index("_check_committed_us_ledger_feed_pin(") + anchored = source.index( + "us_esi_premiums_anchor_feed_mismatches(\n ledger_artifact.facts" + ) + compiled = source.index("target_registry = compile_us_fiscal_target_registry(") + assert checked < anchored < compiled + assert source.count("us_esi_premiums_anchor_feed_mismatches(") == 1 + + # SPM measurement composition refusal # --------------------------------------------------------------------------- diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 416be017a9..d853602880 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "a1e98deaa1f5a451c7d0e1f380a7bf50a82b0cbc94753b885b7cf9a11e7d0005", + "spec_sha256": "b73115661e39ca2263821022aa5c5f50c453a96fb13e8af4e231428962ce0b2e", }, } diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index acfe2ce7ef..94d9d788c0 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe" + "50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index e129234d41..59804cfce5 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -136,6 +136,7 @@ us_educator_expense_signal_gate, us_eligibility_inputs_signal_gate, us_energy_subsidy_signal_gate, + us_esi_premiums_anchor_feed_mismatches, us_esi_premiums_anchor_gate, us_esi_premiums_signal_gate, us_farm_business_income_signal_gate, @@ -12518,6 +12519,16 @@ def _main(argv: Sequence[str] | None = None) -> int | None: manifest_sha256=ledger_artifact.manifest_sha256, allow_unpinned_feed=args.allow_unpinned_feed, ) + # microcosm#454 / d1317: the ESI stage scales to BEA B4923C values it pins + # itself; the feed banks the same series. Refuse a feed that disagrees. + esi_anchor_feed_mismatches = us_esi_premiums_anchor_feed_mismatches( + ledger_artifact.facts + ) + if esi_anchor_feed_mismatches and not args.allow_esi_premium_gaps: + raise SystemExit( + "error: the meps_esi_premiums anchor differs from the Chronicle " + "feed: " + "; ".join(esi_anchor_feed_mismatches) + ) target_registry = compile_us_fiscal_target_registry( ledger_artifact.facts, target_period=PERIOD, diff --git a/tools/build_us_release_input_coverage_manifest.py b/tools/build_us_release_input_coverage_manifest.py index 4a646f724b..b3bc625863 100644 --- a/tools/build_us_release_input_coverage_manifest.py +++ b/tools/build_us_release_input_coverage_manifest.py @@ -119,7 +119,7 @@ "base-builder stage from Census ASEC NOW_OWNGRP / NOW_HIPAID / " "NOW_GRPFTYP2 / PEMLR / NOEMP (restored from the pinned Census person " "files) and MEPS-IC tier x firm-size x State cells, scaled over every " - "policyholder to CMS NHE Table 24. A release must persist it: CBO " + "policyholder to BEA NIPA 7.8 line 17 (B4923C). A release must persist it: CBO " "household market income (gov.household.cbo_market_income_additions) " "and every ESI-to-wages reform bind through it. Required with NO " "reviewed exclusion; absent on pre-#454 bases by construction — the " @@ -380,9 +380,9 @@ "to CBO household market income through " "gov.household.cbo_market_income_additions, its only consumer, so " "neutralizing it lowers cbo_household_market_income by the " - "weighted employer-premium total: about $926 billion at 2024 (the " - "employed policyholders' part of CMS NHE Table 24's $1,047.0 " - "billion, the meps_esi_premiums anchor; the stage measured $925.7 " + "weighted employer-premium total: about $864 billion at 2024 (the " + "employed policyholders' part of BEA NIPA 7.8 line 17's $977.0 " + "billion, the meps_esi_premiums anchor; the stage measured $863.9 " "billion over 74.0 million weighted employed policyholders on the " "default 2023-2025 pool, before calibration). The floor sits at " "about half of that. A structural zero means the column was " diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 64e3cec1b2..319b92afa3 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -61,7 +61,7 @@ MANIFEST_PATH = US_PACKAGE_DIR / "target_parity_manifest.json" FEED_FAMILIES_PATH = US_PACKAGE_DIR / "target_parity_feed_families.json" -DEFAULT_FEED_NAME = "consumer_facts_us_51aa40f.jsonl" +DEFAULT_FEED_NAME = "consumer_facts_us_cf9ece2.jsonl" DEFAULT_FEED_PATH = ( Path.home() / "PolicyEngine" / "_buildh-runtime" / "inputs" / DEFAULT_FEED_NAME ) @@ -239,53 +239,81 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: ), ) -_ESI_ANCHOR_EXCLUSION = ( +_ESI_ORIGIN = ( + "a REAL us-data calibration target: utils/loss.py pinned " + "employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 " + "employer-contribution aggregate; verified at archived commit 2e044571, " + "loss.py line 33, with etl_national_targets carrying the row), removed at " + "9f320d2c ('Remove survey-based national calibration targets', PR #831). " + "The input itself was imputed by the retired eCPS (MEPS-IC priors over " + "ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py " + "L197-271/L1575-1581)." +) + +_ESI_BEA_ANCHOR_EXCLUSION = ( "deferred", - "NHE Table 24 employer contribution to ESI premiums. The meps_esi_premiums " - "base stage (microcosm#454) scales to this fact over the fact's own " - "universe, every current ESI policyholder (NHEA methodology paper 2024, " - "p. 29: active employees, COBRA and retirees), and writes the pe-us input " - "(employer_sponsored_insurance_premiums) for employed policyholders only. " - "So the fact is not a sum target on the column: the column carries about " - "88% of it and the rest is priced, not exported. Validation-gated " - "instead: us_esi_premiums_anchor_gate recomputes the anchor-universe " - "total at release weights and holds it within 5% of this all-employer " - "fact.", + "BEA NIPA Table 7.8 line 17 (B4923C), employer contributions for private " + "group health insurance: the scale anchor of the meps_esi_premiums base " + "stage (microcosm#454, decision d1317). The stage prices every current ESI " + "policyholder from MEPS-IC, scales that universe to this fact (BEA State " + "Personal Income methodology, December 2025, paragraphs 3.18-3.19: " + "contributions for active and retired employees) and writes the pe-us " + "input (employer_sponsored_insurance_premiums) for employed policyholders " + "only. So the fact is not a sum target on the column: the column carries " + "about 88% of it and the rest is priced, not exported. Validation-gated " + "instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total " + "at release weights and holds it within 5% of this fact, and the release " + "tool refuses a feed whose values differ from the stage's pinned anchor " + "(us_esi_premiums_anchor_feed_mismatches).", "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M " - "weighted current policyholders; anchor-universe total $1,047.0B at scale " - "factor 1.040; employed column $925.7B (88.4%); other policyholders " - "$121.3B; private-sector part of the column $713.2B against MEPS-IC's own " + "weighted current policyholders; anchor-universe total $977.0B at scale " + "factor 0.971; employed column $863.9B (88.4%); other policyholders " + "$113.2B; private-sector part of the column $665.6B against MEPS-IC's own " "private active-employee total of $668.6B (2024)", _fence( - origin=( - "a REAL us-data calibration target: utils/loss.py pinned " - "employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA " - "7.8 employer-contribution aggregate; verified at archived " - "commit 2e044571, loss.py line 33, with etl_national_targets " - "carrying the row), removed at 9f320d2c ('Remove survey-based " - "national calibration targets', PR #831). The input itself was " - "imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* " - "policyholder fields, archived 42ed5d45 cps.py " - "L197-271/L1575-1581). The v9.2 feed banks the NHE series for the " - "#454 gate; the NIPA series the us-data target used is not in the " - "feed (source-absent family bea_nipa.private_group_health_insurance)." - ), + origin=_ESI_ORIGIN, purpose=( "anchor the CBO market-income ESI component once the column has a " - "live producer. NHE Table 24 and NIPA 7.8 line 17 both count " - "employer contributions for active and retired employees (NHEA " - "methodology paper; BEA State Personal Income methodology, " - "December 2025, paragraphs 3.18-3.19); NIPA is 6.7% lower in " - "CY2024 and is recorded beside the gate verdict." + "live producer. The BEA annual update of 2026-09-30 revised CY2024 " + "from the $1,002.9B the retired target used to $977.034B." ), verdict_basis=( "deferred, not dropped: the producer is live and scaled to this " "fact, but the column is the employed policyholders' part of it, " "so the fact cannot be compiled as a sum target on the column. " "Release validation (us_esi_premiums_anchor_gate, both dense and " - "sparse exports) holds the anchor-universe total to it. A " - "calibration target needs an exported anchor-universe quantity or " - "a banked active-employee fact." + "sparse exports) holds the anchor-universe total to it. Whether " + "and how to calibrate the column is decision d1264." + ), + ), +) + +_ESI_NHE_EXCLUSION = ( + "deferred", + "NHE Table 24 employer contribution to ESI premiums, a second estimate of " + "the concept the meps_esi_premiums stage scales to (microcosm#454): " + "employer contributions for active employees, COBRA enrollees and " + "retirees (NHEA methodology paper 2024, p. 29). Decision d1317 chose BEA " + "NIPA 7.8 line 17 as the stage's anchor instead; this fact is 7.2% above " + "it for CY2024. It is neither compiled nor gated: " + "us_esi_premiums_anchor_gate records the anchor-universe total against it " + "beside the verdict (cross_check_ratio).", + "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: " + "anchor-universe total $977.0B, 0.933 of NHE's $1,047.0B; scaled to NHE " + "instead, the employed column would be $925.7B", + _fence( + origin=_ESI_ORIGIN, + purpose=( + "a cross-check on the ESI anchor's level: an independent estimate " + "of the same active-plus-retired concept from the national health " + "accounts." + ), + verdict_basis=( + "deferred, not dropped: the stage's anchor is BEA (d1317), and " + "this fact, like it, includes retirees, so it is not a sum target " + "on the employed column. It is recorded beside the anchor " + "verdict. Whether and how to calibrate the column is decision " + "d1264." ), ), ) @@ -293,22 +321,35 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: _ESI_PRIVATE_EMPLOYER_EXCLUSION = ( "deferred", "NHE Table 24 private-employer contribution to ESI premiums, the private " - "part of the all-employer fact the meps_esi_premiums stage scales to " - "(microcosm#454). It is neither compiled nor gated: the stage scales one " - "factor to the all-employer total over every current ESI policyholder, " - "and nothing holds the private-sector part of the column to this fact, " - "which also counts private retirees and COBRA enrollees. The private part " - "is instead recorded beside the anchor verdict against MEPS-IC's own " - "active-employee total (us_esi_premiums_anchor_gate, " - "private_active_cross_check).", + "part of the NHE all-employer fact (microcosm#454). It is neither " + "compiled nor gated: the stage scales one factor to BEA NIPA 7.8 line 17 " + "over every current ESI policyholder, and nothing holds the private-sector " + "part of the column to this fact, which also counts private retirees and " + "COBRA enrollees. The private part is instead recorded beside the anchor " + "verdict against MEPS-IC's own active-employee total " + "(us_esi_premiums_anchor_gate, private_active_cross_check).", "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: " - "private-sector part of the column $713.2B; MEPS-IC private " + "private-sector part of the column $665.6B; MEPS-IC private " "active-employee total $668.6B (2024)", - _ESI_ANCHOR_EXCLUSION[3], + _fence( + origin=_ESI_ORIGIN, + purpose=( + "a sector cross-check that private employers' share of the ESI " + "total is in range." + ), + verdict_basis=( + "deferred, not dropped: no gate reads this fact. The private part " + "of the column is compared with MEPS-IC's active-employee total " + "instead, because this fact includes private retirees and COBRA " + "enrollees. Whether and how to calibrate the column is decision " + "d1264." + ), + ), ) _FAMILY_EXCLUSIONS: dict[str, tuple[str, str, str, dict[str, str]]] = { - "cms_nhe.esi_employer_contribution_premiums": _ESI_ANCHOR_EXCLUSION, + "bea_nipa.private_group_health_insurance": _ESI_BEA_ANCHOR_EXCLUSION, + "cms_nhe.esi_employer_contribution_premiums": _ESI_NHE_EXCLUSION, "cms_nhe.esi_private_employer_contribution_premiums": ( _ESI_PRIVATE_EMPLOYER_EXCLUSION ), @@ -565,31 +606,6 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: # us-data targets the pinned feed carries NO ledger fact for. (reason, evidence, # fence) with classification source_absent. _SOURCE_ABSENT_US_DATA_FAMILIES: dict[str, tuple[str, str, dict[str, str]]] = { - "bea_nipa.private_group_health_insurance": ( - "BEA NIPA Table 7.8 line 17 private group health insurance (B4923C), " - "the employer-contribution aggregate the retired pipeline calibrated " - "employer_sponsored_insurance_premiums to (loss.py 1_002.9e9). The " - "pinned feed carries no fact for it.", - "retired us-data pipeline (archived 2e044571) loss.py line 33; " - "feed bea_nipa families carry no group-health series", - _fence( - origin="us-data loss.py employer_sponsored_insurance_premiums target", - purpose=( - "anchor employer-paid ESI premiums. The series counts " - "contributions for active and retired employees (BEA State " - "Personal Income methodology, December 2025, paragraphs " - "3.18-3.19), the same concept as NHE Table 24." - ), - verdict_basis=( - "source-absent from the feed: the meps_esi_premiums stage pins " - "the BEA file (Section7All_xls.xlsx sha256 de1c34e3..., " - "published 2026-09-30: $977.034B CY2024, revised from " - "$1,002.9B) and records the anchor-universe total against it " - "beside the release anchor verdict; the stage scales to NHE " - "Table 24. Compile it here once the ledger banks the series." - ), - ), - ), "bls.consumer_expenditure": ( "BLS Consumer Expenditure childcare-expense target the retired pipeline " "calibrated (loss.py BLS_CE_TOTALS childcare_expenses = $63.09B). The " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 70ed074fb8..d7fb311ebe 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e02a7b2682963a9d50519b592472e07903379361c63304b63b851575cb0a7dbe" + "50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/spec_engine_coverage.py b/tools/spec_engine_coverage.py index d4b4e88283..16a8e339db 100644 --- a/tools/spec_engine_coverage.py +++ b/tools/spec_engine_coverage.py @@ -41,7 +41,7 @@ REPORT_SCHEMA_VERSION = 3 EXPECTED_POINTER_INVENTORY_SHA256 = ( - "e8d32da2dbdaa015cb4eb363084c66cf509a93dde76cd473113750120dcaef80" + "c1e2a118a0d53db0bb628b14357a2c86db2956332f5b359c9092764f9c451a95" ) DEFAULT_REPORT_PATH = ( Path(__file__).resolve().parents[1] From ebfb05582ae043a5843e5fce6b2ef30cec59d249 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 11 Oct 2026 19:30:45 -0400 Subject: [PATCH 8/8] Answer the fourth review's findings on the BEA anchor (#454) The review approved with no blocking findings; this answers its nits. - Four places still called NHE Table 24 the anchor: the US_DONORS note, the --allow-esi-premium-gaps help, a release-tool comment and a test name. - A waived anchor-versus-feed mismatch left no record. The check is now _check_us_esi_anchor_against_feed: it stops a release build, lets a diagnostic run continue under --allow-unpinned-feed or --allow-esi-premium-gaps, and its result is written to esi_premiums.json (anchor_feed block, schema_version 3). - Nothing in CI tied the anchor to the pinned feed. A test now compares EMPLOYER_PREMIUM_ANCHOR with the three facts the committed re-pin report says the pinned feed added, and with the pin's digest and row count. The release tool's check has direct tests for pass, refuse and each waiver. - The two cms_nhe families shared an origin text that described the BEA series; they have their own. The manifest note uses BEA's wording for what the anchor counts. - The registry comparison now compares ordered lists with target aging on and off (identical; 33,250 distinct names). - Docs: status date, the two checks that hold the anchor to the feed, the waiver wording, and what a later feed re-pin must carry. Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 20 ++--- docs/us-chronicle-feed-repin.md | 13 ++- docs/us-esi-employer-premiums.md | 22 +++-- experiments/us-esi-454/README.md | 6 +- .../feed-repin/compare_registries.py | 62 +++++++++----- .../us-esi-454/feed-repin/registry_diff.json | 28 +++++-- .../build/spec_engine/inventory_coverage.py | 6 +- .../src/microcosm/build/us/source_stages.json | 2 +- .../src/microcosm/build/us/spec/sources.yaml | 20 ++--- .../build/us/target_parity_manifest.json | 6 +- .../microcosm/build/us_runtime/__init__.py | 5 +- .../us/test_us_bundle_core_contracts.py | 2 +- .../engine_free/us/test_us_esi_premiums.py | 42 +++++++++- .../us/test_us_fiscal_refresh_builder.py | 71 +++++++++++++++- .../us/test_us_multispine_pool_tool.py | 2 +- .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_fiscal_refresh_release.py | 80 +++++++++++++++---- tools/build_us_target_parity_manifest.py | 27 ++++--- tools/generate_us_bundle_from_constants.py | 2 +- 19 files changed, 316 insertions(+), 102 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index 286b7a096f..e1b3453e8c 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,7 +4,7 @@ "version": 1 }, "country": "us", - "documentation_sha256": "58a745afcfd8c3fd55317a61c4e79880dffbaf49344888b28b331d0318e67404", + "documentation_sha256": "44f40b4e8aaa2e6be67a7bc9680cc29f88e185c10573ffb9c3f9dfbd3ad1081f", "field_usage": { "authored_normative_field_count": 32437, "claim_count": 49, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "6eb2f999ab2d7b733673f3b13402c01fe6204445b06f8b4f508dc82bbe9e1a3e" + "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "6eb2f999ab2d7b733673f3b13402c01fe6204445b06f8b4f508dc82bbe9e1a3e" + "sha256": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "33e20427d3bddd9ee0b9b55042cc061854442c20cb6c397951642e1d3a334c50", + "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", "stages": 37 }, "failures": [], "observed": { - "sha256": "33e20427d3bddd9ee0b9b55042cc061854442c20cb6c397951642e1d3a334c50", + "sha256": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "d468a00f74ce64da5470df8487f932a8dd5db2b758f58397c6279b3c19fa9b1b" + "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "d468a00f74ce64da5470df8487f932a8dd5db2b758f58397c6279b3c19fa9b1b" + "sha256": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "70c36826e568cc8b2e16687b8ac674618573be09b3b553dd47e5517dd4a17bf2" + "sha256": "fb863f18a79610dc1005089a32e1535c502c5b9f7e5ba589db0d6025b01e3c82" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "b73115661e39ca2263821022aa5c5f50c453a96fb13e8af4e231428962ce0b2e" + "spec_sha256": "0a5ecc7f8da8129ac152720aa0857ec876ba864a807d2da354043b51d9fb3f5c" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "b73115661e39ca2263821022aa5c5f50c453a96fb13e8af4e231428962ce0b2e" + "spec_sha256": "0a5ecc7f8da8129ac152720aa0857ec876ba864a807d2da354043b51d9fb3f5c" }, "status": "pass" } diff --git a/docs/us-chronicle-feed-repin.md b/docs/us-chronicle-feed-repin.md index feb222fb89..24ae473bb2 100644 --- a/docs/us-chronicle-feed-repin.md +++ b/docs/us-chronicle-feed-repin.md @@ -42,9 +42,10 @@ runs). Built at `cf9ece2`: `aggregate_fact_key`) and adds exactly the three B4923C facts: $923,195M, $977,034M and $1,027,929M, each from `Section7All_xls.xlsx` (sha256 `de1c34e3…`); -- compiled at 2024 with the packaged CD crosswalk, aging and the Medicaid - substitutions, all 33,250 targets are identical on both feeds, name, value - and metadata. B4923C is not compiled into a target: the stage scales to it, +- compiled at 2024 with the packaged CD crosswalk and the Medicaid + substitutions, with target aging on and again with it off, the registry is + the same ordered list of targets on both feeds, name, value and metadata + (33,250 targets with aging on). B4923C is not compiled into a target: the stage scales to it, and whether to calibrate the column is decision d1264. The target-parity manifest lists it as a `deferred` reviewed exclusion (82 feed families, 32 compiled, 53 reviewed exclusions; it was a source-absent family before); @@ -53,7 +54,11 @@ runs). Built at `cf9ece2`: The comparison scripts and their reports are in `experiments/us-esi-454/feed-repin/`. The release tool now refuses a feed whose B4923C values differ from the stage's pinned anchor -(`us_esi_premiums_anchor_feed_mismatches`). +(`us_esi_premiums_anchor_feed_mismatches`); a diagnostic run under +`--allow-unpinned-feed` or `--allow-esi-premium-gaps` continues and records +the difference in `esi_premiums.json`. A later re-pin that changes other rows +(microcosm#1192 re-dates the W-2 item facts) must carry the three B4923C +pairs and redo the row comparison against its own previous feed. ## The 2026-10-04 re-pin: TY2023 state AGI bands (microcosm#940) diff --git a/docs/us-esi-employer-premiums.md b/docs/us-esi-employer-premiums.md index d6e38c7a5c..49012ffe25 100644 --- a/docs/us-esi-employer-premiums.md +++ b/docs/us-esi-employer-premiums.md @@ -1,6 +1,6 @@ # US employer-sponsored insurance premiums (#454) -Status as of 2026-10-10. Code: `microcosm.build.us_runtime.esi_premiums` +Status as of 2026-10-11. Code: `microcosm.build.us_runtime.esi_premiums` (stage `meps_esi_premiums`). Receipts: `experiments/us-esi-454/`. ## What the stage produces @@ -190,9 +190,13 @@ that gate. The stage pins BEA's values itself (`EMPLOYER_PREMIUM_ANCHOR`, the Section 7 workbook by URL and SHA-256). The pinned US Chronicle feed banks the same series (record sets `bea_nipa.cy.private_group_health_insurance`, -PolicyEngine/chronicle#331; `docs/us-chronicle-feed-repin.md`), and the -release tool refuses a feed whose values differ from the stage's -(`us_esi_premiums_anchor_feed_mismatches`), so the two cannot drift apart. +PolicyEngine/chronicle#331; `docs/us-chronicle-feed-repin.md`). Two checks +hold them together. A test compares the anchor with the three facts the +committed re-pin report says the pinned feed added, so changing one without +the other fails CI. And the release tool refuses a feed whose values differ +from the stage's (`us_esi_premiums_anchor_feed_mismatches`); a diagnostic run +under `--allow-unpinned-feed` or `--allow-esi-premium-gaps` continues and +records the difference. Two facts differ from what #454 recorded in July: @@ -323,10 +327,12 @@ What the gates need, and what they do not catch: The stage pins the pre-calibration total exactly. Nothing calibrates the column, so the anchor gate is what holds a release's total after reweighting and sparse selection. `esi_premiums.json` is required release evidence: it -carries the anchor verdict, both cross-checks and the base and export signal -verdicts, and both manifests record it. `--allow-esi-premium-gaps` stops the -gates failing a diagnostic build, for example on a base built before this -stage; the failures it waives are still written to that file. +carries the anchor verdict, both cross-checks, the base and export signal +verdicts and the anchor-versus-feed verdict, and both manifests record it. +`--allow-esi-premium-gaps` stops the gates failing a diagnostic build, for +example on a base built before this stage; the failures it waives, including +a feed whose B4923C facts differ from the anchor, are still written to that +file. ## Lineage diff --git a/experiments/us-esi-454/README.md b/experiments/us-esi-454/README.md index 00e80bf535..1947636e30 100644 --- a/experiments/us-esi-454/README.md +++ b/experiments/us-esi-454/README.md @@ -46,5 +46,7 @@ Chronicle feed to `cf9ece2`, which banks BEA B4923C, the stage's anchor `aggregate_fact_key` and compares row bytes; `value_diff.json` records 0 rows missing, 0 changed and the three added B4923C facts. - `compare_registries.py` compiles the US fiscal target registry on both - feeds the way the target-parity generator does; `registry_diff.json` - records 33,250 targets on each and no difference. + feeds, once the way the target-parity generator does (targets aged) and + once with aging off, and compares each as an ordered list; + `registry_diff.json` records 33,250 targets with distinct names on each + feed and identical lists under both settings. diff --git a/experiments/us-esi-454/feed-repin/compare_registries.py b/experiments/us-esi-454/feed-repin/compare_registries.py index b2d3228c40..261cac26d0 100644 --- a/experiments/us-esi-454/feed-repin/compare_registries.py +++ b/experiments/us-esi-454/feed-repin/compare_registries.py @@ -3,15 +3,18 @@ uv run python experiments/us-esi-454/feed-repin/compare_registries.py \ -Uses the same compile as tools/build_us_target_parity_manifest.py (target -period 2024, age targets, canonical congressional-district crosswalk, the -reviewed Medicaid substitutions) and compares every target's name, value and -metadata. +Compiles at target period 2024 with the canonical congressional-district +crosswalk and the reviewed Medicaid substitutions, once the way +tools/build_us_target_parity_manifest.py does (targets aged) and once with +aging off (dollar targets allowed unaged). Each registry is compared as an +ordered list, so a repeated, reordered, added, dropped or changed target shows +up; every field of every target is serialized. """ from __future__ import annotations import dataclasses +import hashlib import importlib.util import json import sys @@ -19,6 +22,11 @@ ROOT = Path(__file__).resolve().parents[3] +CONFIGURATIONS = { + "aged": {"age_targets": True}, + "unaged": {"age_targets": False, "allow_unaged_dollar_targets": True}, +} + def _generator(): path = ROOT / "tools" / "build_us_target_parity_manifest.py" @@ -29,8 +37,7 @@ def _generator(): return module -def _registry(generator, path: Path) -> dict[str, dict]: - facts, _ = generator._load_feed(path) +def _registry(generator, facts: list[dict], **options) -> list[str]: registry = generator.compile_us_fiscal_target_registry( facts, target_period=generator.TARGET_PERIOD, @@ -39,33 +46,46 @@ def _registry(generator, path: Path) -> dict[str, dict]: generator.default_congressional_district_vintage_crosswalk_path() ) ), - age_targets=True, + **options, ) registry, _ = generator.apply_us_medicaid_enrollment_substitutions(registry) - out = {} + rows = [] for spec in registry.specs: fields = ( dataclasses.asdict(spec) if dataclasses.is_dataclass(spec) else vars(spec) ) - out[spec.name] = json.loads(json.dumps(fields, default=str, sort_keys=True)) - return out + rows.append(json.dumps(fields, default=str, sort_keys=True)) + return rows def main() -> None: previous_path, new_path, report_path = (Path(arg) for arg in sys.argv[1:4]) generator = _generator() - previous = _registry(generator, previous_path) - new = _registry(generator, new_path) - report = { - "previous_targets": len(previous), - "new_targets": len(new), - "only_previous": sorted(set(previous) - set(new)), - "only_new": sorted(set(new) - set(previous)), - "changed": sorted(n for n in previous if n in new and previous[n] != new[n]), - } + previous_facts, _ = generator._load_feed(previous_path) + new_facts, _ = generator._load_feed(new_path) + report: dict[str, dict] = {} + differs = False + for label, options in CONFIGURATIONS.items(): + previous = _registry(generator, previous_facts, **options) + new = _registry(generator, new_facts, **options) + names = [json.loads(row)["name"] for row in new] + first_difference = next( + (i for i, (a, b) in enumerate(zip(previous, new, strict=False)) if a != b), + None, + ) + report[label] = { + "options": options, + "previous_targets": len(previous), + "new_targets": len(new), + "distinct_target_names": len(set(names)), + "ordered_lists_identical": previous == new, + "first_differing_index": first_difference, + "registry_sha256": hashlib.sha256("\n".join(new).encode()).hexdigest(), + } + differs = differs or previous != new Path(report_path).write_text(json.dumps(report, indent=1, sort_keys=True) + "\n") - print(json.dumps(report, indent=1)) - if report["only_previous"] or report["only_new"] or report["changed"]: + print(json.dumps(report, indent=1, sort_keys=True)) + if differs: raise SystemExit("the compiled registry differs between the feeds") diff --git a/experiments/us-esi-454/feed-repin/registry_diff.json b/experiments/us-esi-454/feed-repin/registry_diff.json index 71a272d8b0..7164417943 100644 --- a/experiments/us-esi-454/feed-repin/registry_diff.json +++ b/experiments/us-esi-454/feed-repin/registry_diff.json @@ -1,7 +1,25 @@ { - "changed": [], - "new_targets": 33250, - "only_new": [], - "only_previous": [], - "previous_targets": 33250 + "aged": { + "distinct_target_names": 33250, + "first_differing_index": null, + "new_targets": 33250, + "options": { + "age_targets": true + }, + "ordered_lists_identical": true, + "previous_targets": 33250, + "registry_sha256": "054491c95be8741268160682b851ece7e3eb3b1f3a33e968abd09ecac835d171" + }, + "unaged": { + "distinct_target_names": 33250, + "first_differing_index": null, + "new_targets": 33250, + "options": { + "age_targets": false, + "allow_unaged_dollar_targets": true + }, + "ordered_lists_identical": true, + "previous_targets": 33250, + "registry_sha256": "a00aebe49fc492d44bc0b69a4ca4999e2ab5cfe6d423014f02df5e72a0ea58e5" + } } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 91192b4b93..edbc403852 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,18 +350,18 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "d468a00f74ce64da5470df8487f932a8dd5db2b758f58397c6279b3c19fa9b1b", + "full_checkpoint": "e5da85693fab1f1cba9c37beba5cdf91477962d466bc21e9f29125f2f90b7e79", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "6eb2f999ab2d7b733673f3b13402c01fe6204445b06f8b4f508dc82bbe9e1a3e", + "late_resource_semantics": "cbac7c8fbdbb82f0f06ab356d0c9f0d3645ca930550b44af2272d93315f2e934", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", "seed_map": "5641418e4a815da24744d59dfcf82fb11aa2f2ddc290ffcab9627383ea6fa682", "seed_protocol": "fd2021c3a69e2a937c8c748343f1b555a876112a52c887d174d482eea0b1afbb", - "source_manifest": "33e20427d3bddd9ee0b9b55042cc061854442c20cb6c397951642e1d3a334c50", + "source_manifest": "629a0712659a26b2d6aa6c0e5672421f415d625e0599d6baa0a834d118953c6a", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c", } diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index fe90d9ab9f..eee6592225 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -1855,7 +1855,7 @@ "nonnegative_outputs": [ "employer_sponsored_insurance_premiums" ], - "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active employees, COBRA enrollees and retirees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), the anchor decision d1317 chose; the column carries the employed policyholders' part. CMS NHE Table 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees, which the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band)." + "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), the anchor decision d1317 chose; the column carries the employed policyholders' part. CMS NHE Table 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees, which the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band)." }, { "stage": "prior_year_income", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 262556a741..3a0bc01f62 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: 50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24 + sha256: 559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383 stage_manifest: version: 1 country: us @@ -1887,15 +1887,15 @@ stages: notes: 'Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for - active employees, COBRA enrollees and retirees. Column universe: employed (PEMLR 1-2) policyholders - whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated - self-employed and workers without pay carry 0, and their scaled share stays out of the column rather - than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium - and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, - 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier - (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government - and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division - cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 + active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job + has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed + and workers without pay carry 0, and their scaled share stays out of the column rather than being + loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee + contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State + all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 + family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal + employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells + (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 24ca56329e..6ff7939935 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -238,7 +238,7 @@ "reason": "BEA NIPA Table 7.8 line 17 (B4923C), employer contributions for private group health insurance: the scale anchor of the meps_esi_premiums base stage (microcosm#454, decision d1317). The stage prices every current ESI policyholder from MEPS-IC, scales that universe to this fact (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19: contributions for active and retired employees) and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this fact, and the release tool refuses a feed whose values differ from the stage's pinned anchor (us_esi_premiums_anchor_feed_mismatches).", "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $977.0B at scale factor 0.971; employed column $863.9B (88.4%); other policyholders $113.2B; private-sector part of the column $665.6B against MEPS-IC's own private active-employee total of $668.6B (2024)", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (this series, the NIPA 7.8 employer-contribution aggregate, before BEA's 2026-09-30 revision; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", "purpose": "anchor the CBO market-income ESI component once the column has a live producer. The BEA annual update of 2026-09-30 revised CY2024 from the $1,002.9B the retired target used to $977.034B.", "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. Whether and how to calibrate the column is decision d1264." } @@ -463,7 +463,7 @@ "reason": "NHE Table 24 employer contribution to ESI premiums, a second estimate of the concept the meps_esi_premiums stage scales to (microcosm#454): employer contributions for active employees, COBRA enrollees and retirees (NHEA methodology paper 2024, p. 29). Decision d1317 chose BEA NIPA 7.8 line 17 as the stage's anchor instead; this fact is 7.2% above it for CY2024. It is neither compiled nor gated: us_esi_premiums_anchor_gate records the anchor-universe total against it beside the verdict (cross_check_ratio).", "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: anchor-universe total $977.0B, 0.933 of NHE's $1,047.0B; scaled to NHE instead, the employed column would be $925.7B", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "origin": "not itself a us-data calibration target: the retired pipeline calibrated employer_sponsored_insurance_premiums to the BEA NIPA 7.8 series (family bea_nipa.private_group_health_insurance; utils/loss.py 1_002.9e9 at archived commit 2e044571, removed at 9f320d2c, PR #831). The v9.2 feed banked the NHE Table 24 series for the microcosm#454 gate before decision d1317 moved the stage's anchor to BEA.", "purpose": "a cross-check on the ESI anchor's level: an independent estimate of the same active-plus-retired concept from the national health accounts.", "verdict_basis": "deferred, not dropped: the stage's anchor is BEA (d1317), and this fact, like it, includes retirees, so it is not a sum target on the employed column. It is recorded beside the anchor verdict. Whether and how to calibrate the column is decision d1264." } @@ -474,7 +474,7 @@ "reason": "NHE Table 24 private-employer contribution to ESI premiums, the private part of the NHE all-employer fact (microcosm#454). It is neither compiled nor gated: the stage scales one factor to BEA NIPA 7.8 line 17 over every current ESI policyholder, and nothing holds the private-sector part of the column to this fact, which also counts private retirees and COBRA enrollees. The private part is instead recorded beside the anchor verdict against MEPS-IC's own active-employee total (us_esi_premiums_anchor_gate, private_active_cross_check).", "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: private-sector part of the column $665.6B; MEPS-IC private active-employee total $668.6B (2024)", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 employer-contribution aggregate; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "origin": "not itself a us-data calibration target: the retired pipeline calibrated employer_sponsored_insurance_premiums to the BEA NIPA 7.8 series (family bea_nipa.private_group_health_insurance; utils/loss.py 1_002.9e9 at archived commit 2e044571, removed at 9f320d2c, PR #831). The v9.2 feed banked the NHE Table 24 series for the microcosm#454 gate before decision d1317 moved the stage's anchor to BEA.", "purpose": "a sector cross-check that private employers' share of the ESI total is in range.", "verdict_basis": "deferred, not dropped: no gate reads this fact. The private part of the column is compared with MEPS-IC's active-employee total instead, because this fact includes private retirees and COBRA enrollees. Whether and how to calibrate the column is decision d1264." } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py index 50b954a756..60a9e52104 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/__init__.py @@ -2169,8 +2169,9 @@ def to_manifest(self) -> dict[str, object]: notes=( "Current ESI policyholders take the MEPS-IC employer share of " "their tier x firm-size x State (or government census-division) " - "cell, scaled over every policyholder to CMS NHE Table 24 and " - "written for the employed ones (microcosm #454). Deterministic; " + "cell, scaled over every policyholder to BEA NIPA 7.8 line 17 " + "(B4923C) and written for the employed ones (microcosm #454). " + "Deterministic; " "shared by support clones." ), ), diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 33df6af7b8..89d96e39ef 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24", + "sha256": "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py index b45f5e764a..24623916ae 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_esi_premiums.py @@ -393,7 +393,7 @@ def test_private_enrollment_rows_price_the_active_employee_total(self) -> None: class TestEmployerPremium: - def test_weighted_total_over_every_policyholder_is_the_nhe_anchor(self) -> None: + def test_weighted_total_over_every_policyholder_is_the_anchor(self) -> None: result = _run(_realistic_rows(), weights=np.linspace(50, 5_000, 400)) summary = us_esi_premiums_summary(result) assert summary["anchor_universe_employer_total"] == pytest.approx( @@ -1125,6 +1125,46 @@ def test_the_banked_series_matches_the_anchor(self) -> None: } assert esi.us_esi_premiums_anchor_feed_mismatches(self._facts(values)) == [] + def test_the_committed_feed_pin_banks_the_anchor(self) -> None: + """The pinned feed's three added facts are the stage's anchor values. + + The feed itself is too large for the repository, so this reads the + committed row-by-row report of the re-pin. Changing the anchor without + re-pinning the feed, or the reverse, fails here and not only at a + release preflight. + """ + + report = json.loads( + ( + _REPOSITORY_ROOT / "experiments/us-esi-454/feed-repin/value_diff.json" + ).read_text() + ) + pin = json.loads( + files("microcosm.build.us").joinpath("chronicle_feed.json").read_text() + ) + assert report["new"]["sha256"] == pin["facts_sha256"] + assert report["new"]["rows"] == pin["fact_row_count"] + assert report["previous_rows_missing"] == 0 + assert report["previous_rows_changed"] == 0 + added = report["rows_added"] + assert len(added) == 3 + assert {str(row["period"]["value"]): row["value"] for row in added} == dict( + esi.EMPLOYER_PREMIUM_ANCHOR["values"] + ) + template = esi.EMPLOYER_PREMIUM_ANCHOR["feed_record_set"] + assert {row["record_set_id"] for row in added} == { + template.format(year=year) for year in esi.EMPLOYER_PREMIUM_ANCHOR["values"] + } + assert {row["source_sha256"] for row in added} == { + esi.EMPLOYER_PREMIUM_ANCHOR["sha256"] + } + # The same rows, as facts, satisfy the release tool's check. + facts = [ + {"layout": {"record_set_id": row["record_set_id"]}, "value": row["value"]} + for row in added + ] + assert esi.us_esi_premiums_anchor_feed_mismatches(facts) == [] + def test_a_missing_repeated_or_different_fact_is_named(self) -> None: facts = self._facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) facts += self._facts({2023: 923_195_000_000})[:1] diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py index 5584840cf8..fdea4388c2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_fiscal_refresh_builder.py @@ -6417,13 +6417,14 @@ def fake_smoke(*, simulate, period): assert captured["esi_premiums_anchor_periods"] == [builder.PERIOD] signal_evidence = {"evaluated": True, "passed": True, "failures": []} assert json.loads((release_dir / "esi_premiums.json").read_text()) == { - "schema_version": 2, + "schema_version": 3, "enforced": True, "esi_premiums": { "passed": True, "failures": [], "details": {"checked": True}, }, + "anchor_feed": {"enforced": True, "matches": True, "mismatches": []}, "signal": {"base_frame": signal_evidence, "export_frame": signal_evidence}, } for block in (build_manifest, release_manifest["build"]): @@ -16179,11 +16180,75 @@ def test_main_holds_the_esi_anchor_to_the_feed_before_compiling_targets() -> Non source = inspect.getsource(builder._main) checked = source.index("_check_committed_us_ledger_feed_pin(") anchored = source.index( - "us_esi_premiums_anchor_feed_mismatches(\n ledger_artifact.facts" + "_check_us_esi_anchor_against_feed(\n ledger_artifact.facts" ) compiled = source.index("target_registry = compile_us_fiscal_target_registry(") assert checked < anchored < compiled - assert source.count("us_esi_premiums_anchor_feed_mismatches(") == 1 + assert source.count("_check_us_esi_anchor_against_feed(") == 1 + # The same list reaches the evidence file. + assert "_esi_anchor_feed_evidence(\n" in source + + +def _b4923c_facts(values: dict[int, int]) -> list[dict]: + return [ + { + "layout": { + "record_set_id": f"bea_nipa.cy{year}.private_group_health_insurance" + }, + "value": value, + } + for year, value in values.items() + ] + + +def test_the_esi_anchor_feed_check_passes_on_the_banked_series() -> None: + builder = _load_builder_module() + facts = _b4923c_facts( + {2023: 923_195_000_000, 2024: 977_034_000_000, 2025: 1_027_929_000_000} + ) + assert ( + builder._check_us_esi_anchor_against_feed( + facts, allow_unpinned_feed=False, allow_esi_premium_gaps=False + ) + == [] + ) + + +def test_the_esi_anchor_feed_check_refuses_a_feed_that_disagrees() -> None: + """A revised or missing B4923C fact stops a release build.""" + builder = _load_builder_module() + # BEA's pre-revision 2024 value, and no 2025 fact. + facts = _b4923c_facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + with pytest.raises(SystemExit, match="anchor differs from the Chronicle feed"): + builder._check_us_esi_anchor_against_feed( + facts, allow_unpinned_feed=False, allow_esi_premium_gaps=False + ) + + +@pytest.mark.parametrize( + ("allow_unpinned_feed", "allow_esi_premium_gaps"), + [(True, False), (False, True), (True, True)], +) +def test_a_waived_esi_anchor_feed_mismatch_stays_on_the_record( + allow_unpinned_feed, allow_esi_premium_gaps +) -> None: + """A diagnostic flag lets the build go on; the difference is recorded.""" + builder = _load_builder_module() + facts = _b4923c_facts({2023: 923_195_000_000, 2024: 1_002_920_000_000}) + mismatches = builder._check_us_esi_anchor_against_feed( + facts, + allow_unpinned_feed=allow_unpinned_feed, + allow_esi_premium_gaps=allow_esi_premium_gaps, + ) + assert len(mismatches) == 2 + evidence = builder._esi_anchor_feed_evidence(mismatches, enforced=False) + assert evidence == { + "enforced": False, + "matches": False, + "mismatches": mismatches, + } + assert any("1002920000000" in item for item in evidence["mismatches"]) + assert any("cy2025" in item for item in evidence["mismatches"]) # SPM measurement composition refusal diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index d853602880..ed5523b7fa 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -338,7 +338,7 @@ def capture_equality(expected: object, actual: object) -> None: "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "b73115661e39ca2263821022aa5c5f50c453a96fb13e8af4e231428962ce0b2e", + "spec_sha256": "0a5ecc7f8da8129ac152720aa0857ec876ba864a807d2da354043b51d9fb3f5c", }, } diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 94d9d788c0..0f3cd267be 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24" + "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_fiscal_refresh_release.py b/tools/build_us_fiscal_refresh_release.py index 59804cfce5..131d2d1dd4 100644 --- a/tools/build_us_fiscal_refresh_release.py +++ b/tools/build_us_fiscal_refresh_release.py @@ -1311,11 +1311,12 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: help=( "Diagnostic escape hatch (microcosm#454): record the ESI premium " "gates — employer_sponsored_insurance_premiums absent, zero-mass, " - "missing its raw ASEC coverage columns, or off the CMS NHE Table 24 " - "anchor — without failing the build (for example on a base built " - "before the meps_esi_premiums stage). The failures are still " - "written to esi_premiums.json. Release builds must leave this " - "unset." + "missing its raw ASEC coverage columns, or off the BEA NIPA 7.8 " + "line 17 (B4923C) anchor, or a feed whose B4923C facts differ " + "from that anchor — without failing the build (for example on a " + "base built before the meps_esi_premiums stage). The failures are " + "still written to esi_premiums.json. Release builds must leave " + "this unset." ), ) parser.add_argument( @@ -7463,6 +7464,23 @@ def _esi_premiums_signal_evidence(gate: GateResult | None) -> dict[str, object]: } +def _esi_anchor_feed_evidence( + mismatches: Sequence[str], *, enforced: bool +) -> dict[str, object]: + """The anchor-versus-feed verdict for the release evidence file. + + ``mismatches`` is non-empty only when a diagnostic flag waived a feed + whose B4923C facts differ from the stage's anchor; they are kept verbatim + so the waived difference stays on the record. + """ + + return { + "enforced": enforced, + "matches": not mismatches, + "mismatches": list(mismatches), + } + + def _input_mass_reference_gate( base_frame: Frame, *, @@ -12298,6 +12316,35 @@ def _check_committed_us_ledger_feed_pin( ) +def _check_us_esi_anchor_against_feed( + facts: Sequence[Mapping[str, Any]], + *, + allow_unpinned_feed: bool, + allow_esi_premium_gaps: bool, +) -> list[str]: + """Hold the ESI stage's pinned anchor to the feed's banked B4923C facts. + + microcosm#454 / d1317: the ``meps_esi_premiums`` stage scales to BEA + B4923C values it pins itself, and the pinned feed banks the same series. + On the committed pin a difference means the stage drifted from the feed, + so the build stops. A reviewed diagnostic run on another feed + (``--allow-unpinned-feed``) or with the ESI gates waived + (``--allow-esi-premium-gaps``) continues; the differences are returned + either way and written to ``esi_premiums.json``. + """ + + mismatches = us_esi_premiums_anchor_feed_mismatches(facts) + if mismatches and not (allow_unpinned_feed or allow_esi_premium_gaps): + raise SystemExit( + "error: the meps_esi_premiums anchor differs from the Chronicle " + "feed: " + + "; ".join(mismatches) + + "; pass --allow-unpinned-feed or --allow-esi-premium-gaps only " + "for an explicitly reviewed diagnostic run" + ) + return mismatches + + def _main(argv: Sequence[str] | None = None) -> int | None: global _ACTIVE_EMITTER, _ACTIVE_PROGRESS _ACTIVE_EMITTER = None @@ -12519,16 +12566,11 @@ def _main(argv: Sequence[str] | None = None) -> int | None: manifest_sha256=ledger_artifact.manifest_sha256, allow_unpinned_feed=args.allow_unpinned_feed, ) - # microcosm#454 / d1317: the ESI stage scales to BEA B4923C values it pins - # itself; the feed banks the same series. Refuse a feed that disagrees. - esi_anchor_feed_mismatches = us_esi_premiums_anchor_feed_mismatches( - ledger_artifact.facts + esi_anchor_feed_mismatches = _check_us_esi_anchor_against_feed( + ledger_artifact.facts, + allow_unpinned_feed=args.allow_unpinned_feed, + allow_esi_premium_gaps=args.allow_esi_premium_gaps, ) - if esi_anchor_feed_mismatches and not args.allow_esi_premium_gaps: - raise SystemExit( - "error: the meps_esi_premiums anchor differs from the Chronicle " - "feed: " + "; ".join(esi_anchor_feed_mismatches) - ) target_registry = compile_us_fiscal_target_registry( ledger_artifact.facts, target_period=PERIOD, @@ -15380,7 +15422,7 @@ def _main(argv: Sequence[str] | None = None) -> int | None: force_upload=True, ) # microcosm#454: the calibrated ESI premium total over the anchor's - # universe (every policyholder) against CMS NHE Table 24 within tolerance, + # universe (every policyholder) against BEA NIPA 7.8 line 17 within tolerance, # red when the column is absent or zero-mass or the raw ASEC coverage # columns are gone. Runs on the calibrated export for BOTH the dense and # sparse default paths; the verdict ships as the esi_premiums gate @@ -15411,13 +15453,19 @@ def _main(argv: Sequence[str] | None = None) -> int | None: esi_premiums_path.write_text( json.dumps( { - "schema_version": 2, + "schema_version": 3, "enforced": not args.allow_esi_premium_gaps, "esi_premiums": { "passed": esi_premiums_anchor_gate.passed, "failures": list(esi_premiums_anchor_gate.failures), "details": dict(esi_premiums_anchor_gate.details), }, + "anchor_feed": _esi_anchor_feed_evidence( + esi_anchor_feed_mismatches, + enforced=not ( + args.allow_unpinned_feed or args.allow_esi_premium_gaps + ), + ), "signal": { "base_frame": _esi_premiums_signal_evidence( esi_premiums_base_gate diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 319b92afa3..6219fc10af 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -241,13 +241,22 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: _ESI_ORIGIN = ( "a REAL us-data calibration target: utils/loss.py pinned " - "employer_sponsored_insurance_premiums at 1_002.9e9 (the NIPA 7.8 " - "employer-contribution aggregate; verified at archived commit 2e044571, " - "loss.py line 33, with etl_national_targets carrying the row), removed at " - "9f320d2c ('Remove survey-based national calibration targets', PR #831). " - "The input itself was imputed by the retired eCPS (MEPS-IC priors over " - "ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py " - "L197-271/L1575-1581)." + "employer_sponsored_insurance_premiums at 1_002.9e9 (this series, the NIPA " + "7.8 employer-contribution aggregate, before BEA's 2026-09-30 revision; " + "verified at archived commit 2e044571, loss.py line 33, with " + "etl_national_targets carrying the row), removed at 9f320d2c ('Remove " + "survey-based national calibration targets', PR #831). The input itself " + "was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* " + "policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581)." +) + +_ESI_NHE_ORIGIN = ( + "not itself a us-data calibration target: the retired pipeline calibrated " + "employer_sponsored_insurance_premiums to the BEA NIPA 7.8 series (family " + "bea_nipa.private_group_health_insurance; utils/loss.py 1_002.9e9 at " + "archived commit 2e044571, removed at 9f320d2c, PR #831). The v9.2 feed " + "banked the NHE Table 24 series for the microcosm#454 gate before " + "decision d1317 moved the stage's anchor to BEA." ) _ESI_BEA_ANCHOR_EXCLUSION = ( @@ -302,7 +311,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "anchor-universe total $977.0B, 0.933 of NHE's $1,047.0B; scaled to NHE " "instead, the employed column would be $925.7B", _fence( - origin=_ESI_ORIGIN, + origin=_ESI_NHE_ORIGIN, purpose=( "a cross-check on the ESI anchor's level: an independent estimate " "of the same active-plus-retired concept from the national health " @@ -332,7 +341,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "private-sector part of the column $665.6B; MEPS-IC private " "active-employee total $668.6B (2024)", _fence( - origin=_ESI_ORIGIN, + origin=_ESI_NHE_ORIGIN, purpose=( "a sector cross-check that private employers' share of the ESI " "total is in range." diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index d7fb311ebe..1eda413959 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "50c9eb0dc57a74117d17db67f160e500664fed5b3a39c77a423f6e4aaafe0d24" + "559062a1bd10efdd89b8c83da9294fea43062b8b6760524f4350dd84d63ef383" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be"