diff --git a/README.md b/README.md index 105facc379..3992119160 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,13 @@ variables and schema-5 calibration evidence, and requires fresh country/wrapper compatibility checks. It has a local candidate builder and uses the regular publisher's contract with `--parent-h5` and `--preflight-only`. +The non-default ACS local-area chain (`tools/build_us_acs_local_release.py`) +calibrates to the SOI `state` surface by default, the 4,459-target contract of +Build O and Build P; `--soi-mode totals` and `--soi-mode full` are explicit +opt-ins. See +[the ACS local-area SOI target surface](docs/us-acs-local-soi-target-surface.md) +for what each mode contains and where the build records it. + Standard publication uploads the locally built `releases//` artifacts to the Hugging Face dataset, tags the release, and updates `latest.json`. It runs on the build machine (it needs the freshly built H5), so it isn't a CI step: diff --git a/changelog.d/us-acs-local-soi-state-default.changed.md b/changelog.d/us-acs-local-soi-state-default.changed.md new file mode 100644 index 0000000000..cd4ca98a2f --- /dev/null +++ b/changelog.d/us-acs-local-soi-state-default.changed.md @@ -0,0 +1 @@ +Make `state` the default SOI target surface of the US ACS local-area chain (`tools/build_us_acs_local_release.py --soi-mode`, previously `full`): every state-geography `irs_soi` spec outside the congressional-district file, which reproduces the 4,459-target contract Build O and Build P were calibrated to (on the pinned feed, 102 SNAP + 51 Medicaid + 3,819 SOI admin specs from the three TY2022 Historic Table 2 state tables, plus 487 population marginals). `totals` (ACA premium tax credit rows only; no state AGI, income-tax or EITC total) and `full` (adds the TY2023 congressional-district file; too large for a dense matrix on one machine) are explicit opt-ins. Unknown modes are refused, the materialized mode is recorded in the build manifest (`materialize.soi_mode`) and the release refresh recipe, and a checkpoint that records no known mode is refused at package. The national release is unchanged. diff --git a/docs/us-acs-local-soi-target-surface.md b/docs/us-acs-local-soi-target-surface.md new file mode 100644 index 0000000000..02e71bad31 --- /dev/null +++ b/docs/us-acs-local-soi-target-surface.md @@ -0,0 +1,158 @@ +# US ACS local-area SOI target surface + +`tools/build_us_acs_local_release.py --stage materialize` calibrates the ACS +local-area artifact to a state-level administrative surface: USDA SNAP, +CMS Medicaid enrollment and IRS SOI. `--soi-mode` decides which SOI specs it +keeps. + +| `--soi-mode` | What it keeps | Status | +|---|---|---| +| `state` | State-level `irs_soi` specs at `ledger_geography_level == "state"` whose `ledger_layout_record_set_spec_id` is not a congressional-district file, of any `target_role` | Default | +| `totals` | State-level `irs_soi` specs whose `target_role` is not `soi_fiscal_distribution` | Explicit opt-in | +| `full` | Every state-level `irs_soi` spec | Explicit opt-in | + +"State-level" means the spec's metadata carries `state_fips`, which the +congressional-district SOI rows also carry. The rule lives in +`soi_surface_predicate`; `state_admin_specs` applies it after the production +compile (`compile_us_fiscal_target_registry(age_targets=True)` and the RI +Medicaid substitution). `state` does not select a spec that lacks either +`ledger_geography_level` or `ledger_layout_record_set_spec_id`, since it cannot +tell which contract such a spec belongs to. An unknown mode is refused before +the feed is read. The national release does not use this switch: +`tools/build_us_fiscal_refresh_release.py` has no SOI mode and does not call +`state_admin_specs`. + +## Why `state` is the default + +`state` is the SOI contract every earlier ACS local-area release was calibrated +to. Build O (`populace-us-2024-buildo-acs-local-77e2061-20260724T110908Z`) +selected it as `full` while `state_admin_specs` compiled with +`include_congressional_district_targets=False` (populace `77e2061`, +`tools/build_us_acs_local_release.py:151-154`). Build P's ACS local release +(`populace-us-2024-buildp-acs-local-592ae5d6-20260819T020303Z`) calibrated to +the same 4,459 targets. Commit `b7922b089` (2026-08-21, "compile one target +surface unconditionally") removed that switch, so `full` grew to include the +TY2023 congressional-district SOI file and no mode reproduced the historical +contract until `state`. + +Max chose `state` as the default on 2026-09-22, after the measurements below +showed that `totals` carries no state AGI, income-tax or EITC total and that +`full` does not fit one 128 GB machine. It is the only surface that keeps the +state income targets and fits, and it lets a new local build be scored against +the published Build O on the same targets. Before that day the default was +`full`; for part of 2026-09-22, on this branch only, it was `totals`. + +`state` depends on the restated-filter support of #969 (merged 2026-09-22): +without it the materializer's Ledger-filter guard refuses 1,014 of the +surface's specs. + +## The surfaces on the pinned feed + +Measured on 2026-09-22 by calling `state_admin_specs(feed, ["snap", +"medicaid", "soi"], soi_mode=...)` on the pinned +`consumer_facts_us_c5e5bf8.jsonl` (sha256 `b8543739…`, the `facts_sha256` in +`packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json`). The +Chronicle `b571381` consumer artifact gives the same counts. + +| Family | `state` | `totals` | `full` | +|---|---:|---:|---:| +| `usda_snap` | 102 | 102 | 102 | +| `cms_medicaid` (enrollment) | 51 | 51 | 51 | +| `irs_soi` | 3,819 | 607 | 30,913 | +| **Admin specs** | **3,972** | **760** | **31,066** | + +The 487 population marginals (51 states and 436 congressional districts) are +added on top in every mode, so `state` calibrates to 4,459 targets: Build P's +set, and Build O's 4,461 minus the Vermont under-$1 taxable-interest pair the +compiler now excludes as unsupported (`US_FISCAL_TARGET_SUPPORT_EXCLUSIONS` in +`us_runtime/fiscal_targets.py`). + +### What `state` contains + +All 3,819 SOI specs sit at state geography and come from the three TY2022 +Historic Table 2 state tables: + +| Record set spec | Specs | Content | +|---|---:|---| +| `irs_soi.historic_table_2.state_broad_totals.v1` | 2,397 | 47 all-income-range measures x 51 states, including AGI, income tax, and ACA premium tax credit returns and amounts | +| `irs_soi.historic_table_2.state_agi_counts_and_amounts.v1` | 912 | taxable interest by AGI band | +| `irs_soi.historic_table_2.state_eitc.v1` | 510 | EITC returns and amounts by number of qualifying children | + +It holds no congressional-district SOI row and no row from the TY2023 +congressional-district file. The feed-gated test +`test_pinned_feed_soi_surfaces_match_their_contracts` pins these counts. + +### What `totals` contains, and what it does not + +All 607 SOI specs that `totals` keeps are ACA premium tax credit measures (547 +returns, 60 amounts), 454 of them at congressional-district geography. +`_soi_target_role` in `us_runtime/fiscal_targets.py` gives named roles only to +country-level all-income-range facts, the all-income-range PTC measures and +the W-2 tip items; every other SOI fact gets `soi_fiscal_distribution`, +whether or not it is an AGI band. **`totals` therefore carries no state or +district AGI, income-tax or EITC total.** + +### What `full` adds + +`full`'s 30,306 `soi_fiscal_distribution` specs split by +`ledger_geography_level` and by whether an AGI bound is finite: + +| Geography | Income range | Geographies | SOI variables | Specs | +|---|---|---:|---:|---:| +| Congressional district | All | 436 | 25 | 23,886 | +| State | All | 51 | 28 | 5,508 | +| State | AGI band | 51 | 1 (`taxable_interest_income`) | 912 | + +Its state all-income-range rows include the same state concepts twice, once +from the TY2022 Historic Table 2 and once from the TY2023 congressional-district +file (`_total` rows); the two copies disagree by a few percent after both +are aged to 2024. Its district rows reconcile to their state parents in the +same file. Using them needs one vintage per state concept and a sparse target +matrix. + +### Matrix size + +This is arithmetic from the code, not a memory measurement. +`materialize_chunked` allocates one float32 column per admin spec for every +household (`np.memmap(..., dtype=np.float32, shape=(n_households, +len(names)))`), and `write_lean_checkpoint` copies that memmap into memory +(`np.array(admin_matrix)`). At 1,588,854 households, 4 bytes x households x +specs gives: + +| Surface | Admin specs | Dense float32 admin matrix | +|---|---:|---:| +| `totals` | 760 | 4.5 GiB | +| `state` (default) | 3,972 | 23.5 GiB | +| `full` | 31,066 | 183.9 GiB | + +Measured peaks for comparison: Build P's ACS local release, on this surface, +recorded a 93.9 GB materialize peak (`build_manifest.json` → +`materialize.peak_rss_gb`); the 2026-09-22 hours rebuild on `totals` peaked at +75.6 GB in materialize. `full` does not fit a 128 GB machine as a dense matrix. + +## Where the chosen mode is recorded + +- `materialize_rss.json` in the checkpoint directory (`soi_mode`), written by + `--stage materialize`. Later stages never re-select targets; they read this + file, whatever `--soi-mode` they are given. +- `gate_summary.json`, under `gates.calibration.calibrated_surface.soi_mode` + (finalize), which `build_manifest.json` also embeds under `gates`. +- `build_manifest.json`, under `materialize.soi_mode` (package). +- The `refresh_recipe.release` command in `build_manifest.json` and + `release_manifest.json`, which names `--soi-mode `, so re-running it + reproduces the recorded surface even if the default changes again. + +`--stage package` refuses a checkpoint whose `materialize_rss.json` does not +record one of the three modes, before any release directory exists. + +## Operating notes + +- The default command calibrates to `state`. Raise the supervisor's RSS cap + above Build P's 93.9 GB peak before a full-scale run, and leave room on disk + for the 23.5 GiB memmap. +- To reproduce the 2026-09-22 hours rebuild (#974), pass `--soi-mode totals`. + To reproduce a build that ran as `full` after `b7922b089`, pass + `--soi-mode full`. +- A checkpoint materialized under an earlier default records its own mode and + packages as that mode. +- `--soi-mode` only matters when `soi` is in `--families`. diff --git a/packages/microcosm-build/tests/test_us_acs_local_release_tool.py b/packages/microcosm-build/tests/test_us_acs_local_release_tool.py index 0fb50bfa4b..1d3349b508 100644 --- a/packages/microcosm-build/tests/test_us_acs_local_release_tool.py +++ b/packages/microcosm-build/tests/test_us_acs_local_release_tool.py @@ -3,9 +3,12 @@ from __future__ import annotations import importlib.util +import inspect import json +import os import shlex from pathlib import Path +from types import SimpleNamespace import numpy as np import pandas as pd @@ -316,7 +319,11 @@ def _package_evidence_args(module, tmp_path: Path, monkeypatch, *, hours_report) "consumer_export.json": {"staging_sha256": module._sha256(staging)}, "held_back_columns.json": {"total": 0}, "reviewed_null_fills.json": {"columns_filled": []}, - "materialize_rss.json": {"materialize_peak_rss_gb": 1.0, "hh_chunk": 1}, + "materialize_rss.json": { + "soi_mode": "totals", + "materialize_peak_rss_gb": 1.0, + "hh_chunk": 1, + }, "consumer_reviewed_null_fills.json": {"columns_filled": []}, } for name, payload in evidence.items(): @@ -439,14 +446,19 @@ def load_frame(path, *_a, **_k): _UNSET = object() -def _package_args_before_evidence(module, tmp_path: Path, *, max_households=_UNSET): +def _package_args_before_evidence( + module, tmp_path: Path, *, max_households=_UNSET, soi_mode="totals" +): """The package stage's inputs up to (not including) the qa/consumer evidence. ``max_households`` is what the staging summary records under - ``orchestration``; the sentinel omits the block entirely. + ``orchestration``; the sentinel omits the block entirely. ``soi_mode`` is + what ``materialize_rss.json`` records; the sentinel omits the file. """ ckpt = tmp_path / "ckpt" ckpt.mkdir() + if soi_mode is not _UNSET: + (ckpt / "materialize_rss.json").write_text(json.dumps({"soi_mode": soi_mode})) staging = tmp_path / "staging.h5" staging.write_bytes(b"staging") summary: dict = {} @@ -537,6 +549,7 @@ def test_do_package_requires_qa_and_consumer_evidence(tmp_path: Path) -> None: out_h5.write_bytes(b"artifact") (ckpt / "calibration_diagnostics.json").write_text(json.dumps({"households": 1})) (ckpt / "gate_summary.json").write_text(json.dumps({"gates": {}})) + (ckpt / "materialize_rss.json").write_text(json.dumps({"soi_mode": "totals"})) (ckpt / "run_identity.json").write_text( json.dumps( { @@ -578,6 +591,255 @@ def test_do_package_requires_qa_and_consumer_evidence(tmp_path: Path) -> None: module.do_package(args) +# --------------------------------------------------------------------------- +# SOI target surface: state (Build O contract) by default; totals and full opt-in +# --------------------------------------------------------------------------- + + +def _materialize_argv(tmp_path: Path, *extra: str) -> list[str]: + return [ + "--stage", + "materialize", + "--staging-h5", + str(tmp_path / "staging.h5"), + "--checkpoint-dir", + str(tmp_path / "ckpt"), + "--feed", + str(tmp_path / "facts.jsonl"), + *extra, + ] + + +def test_soi_mode_defaults_to_state_and_totals_and_full_are_explicit_opt_ins( + tmp_path: Path, +) -> None: + """Max's ruling of 2026-09-22: the ACS local default is Build O's + state-geography SOI contract; ``totals`` and ``full`` are reachable only by + asking for them.""" + + module = _load_tool_module() + assert module.SOI_MODES == ("state", "totals", "full") + assert module.DEFAULT_SOI_MODE == module.SOI_MODE_STATE == "state" + assert module._parse_args(_materialize_argv(tmp_path)).soi_mode == "state" + for mode in ("totals", "full"): + assert ( + module._parse_args(_materialize_argv(tmp_path, "--soi-mode", mode)).soi_mode + == mode + ) + signature = inspect.signature(module.state_admin_specs) + assert signature.parameters["soi_mode"].default == "state" + with pytest.raises(SystemExit): + module._parse_args(_materialize_argv(tmp_path, "--soi-mode", "ful")) + + +_HT2_BROAD = "irs_soi.historic_table_2.state_broad_totals.v1" +_HT2_AGI = "irs_soi.historic_table_2.state_agi_counts_and_amounts.v1" +_CD_FILE = "irs_soi.congressional_district_2022.all_returns.v1" + + +def _spec( + role: str | None, + *, + state: bool = True, + geography: str | None = "state", + record_set: str | None = _HT2_BROAD, +) -> SimpleNamespace: + metadata: dict[str, str] = {} + if state: + metadata["state_fips"] = "06" + if role is not None: + metadata["target_role"] = role + if geography is not None: + metadata["ledger_geography_level"] = geography + if record_set is not None: + metadata["ledger_layout_record_set_spec_id"] = record_set + return SimpleNamespace(metadata=metadata) + + +def test_state_soi_surface_is_build_o_contract_by_record_set_and_geography() -> None: + """``state`` keeps state-geography specs outside the district file, of any + role (AGI-band rows included), and refuses to guess on missing metadata.""" + + module = _load_tool_module() + state = module.soi_surface_predicate("state") + + for role in ("soi_fiscal_distribution", "aca_ptc_returns", "aca_spending", None): + assert state(_spec(role)), role + assert state(_spec(role, record_set=_HT2_AGI)), role + # The TY2023 congressional-district file is excluded even at state geography + # (its _total rows restate the Historic Table 2 totals a year later). + assert not state(_spec("soi_fiscal_distribution", record_set=_CD_FILE)) + assert not state(_spec("aca_ptc_returns", record_set=_CD_FILE)) + # District geography is excluded even from Historic Table 2. + assert not state( + _spec("soi_fiscal_distribution", geography="congressional_district") + ) + # No state_fips: never on the state surface. + assert not state(_spec("soi_fiscal_distribution", state=False)) + # Missing either key: the mode cannot tell which contract the spec is in. + assert not state(_spec("soi_fiscal_distribution", geography=None)) + assert not state(_spec("soi_fiscal_distribution", record_set=None)) + assert not state(_spec("soi_fiscal_distribution", record_set="")) + + +def test_soi_surface_predicate_drops_soi_fiscal_distribution_only_in_totals() -> None: + module = _load_tool_module() + totals = module.soi_surface_predicate("totals") + full = module.soi_surface_predicate("full") + + band = _spec("soi_fiscal_distribution") + assert not totals(band) + assert full(band) + for role in ("aca_ptc_returns", "aca_spending", None): + assert totals(_spec(role)) and full(_spec(role)), role + # Neither mode reaches past the state surface. + for mode_predicate in (totals, full): + assert not mode_predicate(_spec("aca_spending", state=False)) + assert not mode_predicate(_spec("soi_fiscal_distribution", state=False)) + + +def test_unknown_soi_mode_is_refused_before_the_feed_is_read(tmp_path: Path) -> None: + """A typo must never fall through to either surface (the old predicate + treated every value other than ``full`` as totals).""" + + module = _load_tool_module() + missing_feed = tmp_path / "never-read.jsonl" + for call in ( + lambda: module.soi_surface_predicate("ful"), + lambda: module.state_admin_specs(missing_feed, ["soi"], soi_mode="ful"), + lambda: module.release_refresh_recipe("ful"), + ): + with pytest.raises(ValueError, match="soi_mode must be one of"): + call() + + +@pytest.mark.parametrize("soi_mode", ["state", "totals", "full"]) +def test_release_refresh_recipe_reproduces_its_soi_mode( + tmp_path: Path, soi_mode: str +) -> None: + """The recipe names the mode, so re-running it cannot drift with the + parser default.""" + + module = _load_tool_module() + recipe = shlex.split(module.release_refresh_recipe(soi_mode)) + assert recipe[:3] == ["uv", "run", "tools/build_us_acs_local_release.py"] + assert recipe[recipe.index("--soi-mode") + 1] == soi_mode + args = module._parse_args(recipe[3:]) + assert args.soi_mode == soi_mode + assert args.stages == ["materialize", "calibrate", "qa", "finalize", "package"] + + +@pytest.mark.parametrize( + ("recorded", "message"), + [ + (_UNSET, r"records soi_mode=None"), + ("bands", r"records soi_mode='bands'"), + ], + ids=["not-recorded", "unknown"], +) +def test_package_refuses_a_checkpoint_without_a_known_soi_mode( + tmp_path: Path, recorded, message +) -> None: + module = _load_tool_module() + args = _package_args_before_evidence( + module, tmp_path, max_households=None, soi_mode=recorded + ) + with pytest.raises(SystemExit, match=message): + module.do_package(args) + assert not (args.out / "releases").exists(), "a refusal leaves no release" + + +@pytest.mark.parametrize("recorded", ["state", "totals", "full"]) +def test_package_records_the_materialized_soi_mode_not_the_parser_default( + tmp_path: Path, monkeypatch, recorded: str +) -> None: + """The package invocation passes no ``--soi-mode`` (so the parser says + ``state``); the manifest and recipe must carry what materialize used.""" + + module = _load_tool_module() + args = _package_evidence_args( + module, + tmp_path, + monkeypatch, + hours_report={"passed": True, "failures": [], "detail": {}}, + ) + assert args.soi_mode == "state" + (args.checkpoint_dir / "materialize_rss.json").write_text( + json.dumps({"soi_mode": recorded, "hh_chunk": 1}) + ) + + result = module.do_package(args) + + release_dir = Path(result["release_dir"]) + build_manifest = json.loads((release_dir / "build_manifest.json").read_text()) + release_manifest = json.loads((release_dir / "release_manifest.json").read_text()) + assert build_manifest["materialize"]["soi_mode"] == recorded + for manifest in (build_manifest, release_manifest): + recipe = shlex.split(manifest["refresh_recipe"]["release"]) + assert recipe[recipe.index("--soi-mode") + 1] == recorded + + +def test_pinned_feed_soi_surfaces_match_their_contracts() -> None: + """On the real feed: the default ``state`` surface is Build O/P's + 3,972-spec admin contract, built only from the three Historic Table 2 + state tables at state geography; ``totals`` holds no + ``soi_fiscal_distribution`` spec; ``full`` adds exactly those specs to + ``totals`` and contains ``state``. + + The pinned consumer-facts feed is a 164 MB public aggregate export that + no CI lane carries, so this runs only when ``MICROCOSM_US_CHRONICLE_FACTS`` + points at it (the convention of the #969 state-surface arm). Three + registry compiles; about 3 GB peak RSS. + """ + + feed = os.environ.get("MICROCOSM_US_CHRONICLE_FACTS", "") + if not feed or not Path(feed).exists(): + pytest.skip( + "set MICROCOSM_US_CHRONICLE_FACTS to the pinned consumer-facts feed" + ) + + module = _load_tool_module() + families = ["snap", "medicaid", "soi"] + state_registry, _ = module.state_admin_specs(feed, families) + totals_registry, _ = module.state_admin_specs(feed, families, soi_mode="totals") + full_registry, _ = module.state_admin_specs(feed, families, soi_mode="full") + state_specs = state_registry.specs + totals_names = {spec.name for spec in totals_registry.specs} + full_by_name = {spec.name: spec for spec in full_registry.specs} + + # Build P's ACS local contract: 102 SNAP + 51 Medicaid + 3,819 SOI admin + # specs (4,459 with the 487 population marginals); Build O's 4,461 plus the + # Vermont under-$1 taxable-interest pair the compiler now excludes. + families_count = {} + for spec in state_specs: + families_count[spec.family] = families_count.get(spec.family, 0) + 1 + assert families_count == {"usda_snap": 102, "cms_medicaid": 51, "irs_soi": 3819} + soi = [spec for spec in state_specs if spec.family == "irs_soi"] + record_sets = {} + for spec in soi: + key = spec.metadata["ledger_layout_record_set_spec_id"] + record_sets[key] = record_sets.get(key, 0) + 1 + assert record_sets == { + "irs_soi.historic_table_2.state_broad_totals.v1": 2397, + "irs_soi.historic_table_2.state_agi_counts_and_amounts.v1": 912, + "irs_soi.historic_table_2.state_eitc.v1": 510, + } + assert {spec.metadata["ledger_geography_level"] for spec in soi} == {"state"} + assert {spec.name for spec in state_specs} <= set(full_by_name) + + assert not [ + spec + for spec in totals_registry.specs + if spec.metadata.get("target_role") == "soi_fiscal_distribution" + ] + assert totals_names < set(full_by_name) + added = [full_by_name[name] for name in set(full_by_name) - totals_names] + assert added + assert {(spec.family, spec.metadata.get("target_role")) for spec in added} == { + ("irs_soi", "soi_fiscal_distribution") + } + + def _staging_frame_with_hours(weekly: list[float], last_week: list[float]): """A minimal US-schema staging frame carrying the two pool hours columns.""" @@ -636,6 +898,9 @@ def _finalize_args(module, tmp_path: Path): {"final_loss": 0.1, "initial_loss": 0.5, "mass_conserved_ratio": 1.0} ) ) + # Every checkpoint materialize writes records its SOI mode; package + # refuses one that does not. + (ckpt / "materialize_rss.json").write_text(json.dumps({"soi_mode": "totals"})) ladder = tmp_path / "ladder.npz" ladder.write_bytes(b"ladder-bytes") return module._parse_args( diff --git a/tools/build_us_acs_local_release.py b/tools/build_us_acs_local_release.py index 4fc80cbf05..076fbeb455 100644 --- a/tools/build_us_acs_local_release.py +++ b/tools/build_us_acs_local_release.py @@ -14,7 +14,9 @@ Ledger feed exactly like the production path (``compile_us_fiscal_target_registry`` -> RI Medicaid substitution -> state {usda_snap, cms_medicaid[enrollment], - irs_soi}), run the household-chunked engine pass under the + irs_soi}; ``--soi-mode state`` by default -- Build O's + state-geography SOI contract -- with ``totals`` and ``full`` + as explicit opt-ins), run the household-chunked engine pass under the nullable-artifact contract (input-schema projection + reviewed-null fill), add PUMA-ladder population marginals (state + congressional district), and write a lean float32 @@ -84,6 +86,62 @@ "--puma-ladder build/us/us_puma_ladder_2020.npz" ) +#: ``--soi-mode`` values for the state-level ``irs_soi`` surface. +#: +#: ``state`` (the default; Max's ruling of 2026-09-22) is the SOI contract every +#: earlier ACS local-area release was calibrated to, Build O's and Build P's: +#: every ``irs_soi`` spec at state geography whose Ledger record set is not a +#: congressional-district file -- on the pinned feed, the three TY2022 Historic +#: Table 2 state tables (broad totals, AGI bands, EITC by qualifying +#: children). Build O selected it as ``full`` with congressional-district +#: targets switched off (``include_congressional_district_targets=False`` at +#: populace@77e2061); commit b7922b089 removed that switch, so no mode +#: reproduced it until this one. +#: +#: ``totals`` keeps only the specs whose ``target_role`` is not +#: ``soi_fiscal_distribution`` -- on the pinned feed, ACA premium tax credit +#: rows and no state AGI, income-tax or EITC total. ``full`` keeps every +#: state-bearing spec, including the TY2023 congressional-district file, which +#: needs a dense matrix too large for one 128 GB machine. Both are explicit +#: opt-ins. docs/us-acs-local-soi-target-surface.md has the measured surfaces. +SOI_MODE_STATE = "state" +SOI_MODE_TOTALS = "totals" +SOI_MODE_FULL = "full" +SOI_MODES = (SOI_MODE_STATE, SOI_MODE_TOTALS, SOI_MODE_FULL) +DEFAULT_SOI_MODE = SOI_MODE_STATE +#: Ledger record-set specs from a congressional-district file start with this; +#: ``state`` mode excludes them, which is what Build O's switch did. +CONGRESSIONAL_DISTRICT_RECORD_SET_SPEC_PREFIX = "irs_soi.congressional_district_" + + +def _require_soi_mode(soi_mode: str) -> str: + if soi_mode not in SOI_MODES: + raise ValueError( + f"soi_mode must be one of {SOI_MODES}, got {soi_mode!r}; an " + "unrecognised mode must never fall through to either surface." + ) + return soi_mode + + +def release_refresh_recipe(soi_mode: str) -> str: + """The one-command release refresh, pinned to the SOI surface it built. + + The recipe names ``--soi-mode`` explicitly so re-running it reproduces + the recorded surface even if the parser default changes again. + """ + + _require_soi_mode(soi_mode) + return ( + "uv run tools/build_us_acs_local_release.py --stage all " + "--staging-h5 /acs_multispine_staging.h5 " + "--feed --feed-sha256 " + f"--soi-mode {soi_mode} " + "--ladder build/us/us_puma_ladder_2020.npz " + "--checkpoint-dir /checkpoints " + "--out-h5 /populace_us_2024_acs_local.h5 " + "--out /release" + ) + # --------------------------------------------------------------------------- # Shared telemetry @@ -137,16 +195,63 @@ def _load_json(path: Path) -> dict: # --------------------------------------------------------------------------- -def state_admin_specs(feed: str | Path, families: list[str], soi_mode: str = "full"): +def soi_surface_predicate(soi_mode: str): + """The ``irs_soi`` spec filter for one ``--soi-mode``. + + Every mode keeps only specs carrying ``state_fips`` (state rows, and the + congressional-district rows that also carry their state). + + - ``state`` then keeps a spec only at ``ledger_geography_level == + "state"`` whose ``ledger_layout_record_set_spec_id`` is not a + congressional-district file, of any ``target_role`` (AGI-band rows + included). A spec missing either key is not selected: the mode cannot + tell which contract it belongs to. + - ``totals`` drops every ``soi_fiscal_distribution`` spec (the + jetsam-safe Option B of the Build L runbook). That role is not only + AGI-band slices: every SOI fact without a named role gets it, which at + state level includes the all-income-range state and district rows. + - ``full`` keeps them all. + """ + + _require_soi_mode(soi_mode) + + def selected(spec) -> bool: + metadata = spec.metadata + if "state_fips" not in metadata: + return False + if soi_mode == SOI_MODE_STATE: + record_set_spec = metadata.get("ledger_layout_record_set_spec_id") + return ( + metadata.get("ledger_geography_level") == "state" + and isinstance(record_set_spec, str) + and bool(record_set_spec) + and not record_set_spec.startswith( + CONGRESSIONAL_DISTRICT_RECORD_SET_SPEC_PREFIX + ) + ) + return ( + soi_mode == SOI_MODE_FULL + or metadata.get("target_role") != "soi_fiscal_distribution" + ) + + return selected + + +def state_admin_specs( + feed: str | Path, families: list[str], soi_mode: str = DEFAULT_SOI_MODE +): """Select the state-level admin surface from the production compile path. feed -> ``compile_us_fiscal_target_registry(age_targets=True)`` -> ``apply_us_medicaid_enrollment_substitutions`` (RI FIPS-44) -> state-level - {usda_snap, cms_medicaid[enrollment], irs_soi}. ``soi_mode='totals'`` - drops the ``soi_fiscal_distribution`` AGI-band slices (the jetsam-safe - Option B of the Build L runbook); ``'full'`` keeps them. + {usda_snap, cms_medicaid[enrollment], irs_soi}. The SOI slice follows + :func:`soi_surface_predicate`: ``state`` (the default), ``totals`` or + ``full``. """ + # Refuse an unknown mode before loading the feed and compiling the registry. + _require_soi_mode(soi_mode) + from microcosm.build.ledger_artifact import load_ledger_consumer_artifact from microcosm.build.us_runtime import ( default_congressional_district_vintage_crosswalk_path, @@ -188,17 +293,9 @@ def state_level(spec) -> bool: ), ).specs if "soi" in families: - soi_predicate = ( - state_level - if soi_mode == "full" - else ( - lambda spec: ( - state_level(spec) - and spec.metadata.get("target_role") != "soi_fiscal_distribution" - ) - ) - ) - picked += registry.select(family="irs_soi", predicate=soi_predicate).specs + picked += registry.select( + family="irs_soi", predicate=soi_surface_predicate(soi_mode) + ).specs return TargetRegistry(list(picked), country="us"), ri_substitutions @@ -1706,6 +1803,25 @@ def _require_uncapped_staging(staging_summary: dict) -> dict[str, object]: return {key: orchestration.get(key) for key in _STAGING_ORCHESTRATION_KEYS} +def _require_recorded_soi_mode(materialize_rss: dict) -> str: + """The SOI surface the checkpoint was materialized with, or refuse. + + Later stages never re-select targets, so the mode that counts is the one + ``--stage materialize`` recorded, not this invocation's ``--soi-mode``. + A checkpoint that does not record a known mode cannot say which surface + was calibrated, and its refresh recipe could not reproduce it. + """ + + soi_mode = materialize_rss.get("soi_mode") + if soi_mode not in SOI_MODES: + raise SystemExit( + f"materialize_rss.json records soi_mode={soi_mode!r}, not one of " + f"{SOI_MODES}; packaging cannot record which SOI surface was " + "calibrated. Re-run --stage materialize with the current tool." + ) + return soi_mode + + def do_package(args) -> dict: diagnostics = _load_json(args.checkpoint_dir / "calibration_diagnostics.json") gate_report = _load_json(args.gate_report) @@ -1725,6 +1841,7 @@ def do_package(args) -> dict: ) # Before any release directory exists: a refused smoke leaves nothing. staging_orchestration = _require_uncapped_staging(staging_summary) + soi_mode = _require_recorded_soi_mode(materialize_rss) code = _repo_code_identity(args.allow_dirty) timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") @@ -1839,15 +1956,7 @@ def _version(package: str) -> str: "unchanged." ), "staging": LEGACY_STAGING_REFRESH_RECIPE, - "release": ( - "uv run tools/build_us_acs_local_release.py --stage all " - "--staging-h5 /acs_multispine_staging.h5 " - "--feed --feed-sha256 " - "--ladder build/us/us_puma_ladder_2020.npz " - "--checkpoint-dir /checkpoints " - "--out-h5 /populace_us_2024_acs_local.h5 " - "--out /release" - ), + "release": release_refresh_recipe(soi_mode), "publish": ( "tools/publish_release.sh --no-latest " f"--artifact-root --repo-id {HF_REPO_ID}" @@ -1895,6 +2004,7 @@ def _version(package: str) -> str: ) }, "materialize": { + "soi_mode": soi_mode, "peak_rss_gb": materialize_rss.get("materialize_peak_rss_gb"), "hh_chunk": materialize_rss.get("hh_chunk"), "engine_pass": ( @@ -2081,7 +2191,21 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: ) parser.add_argument("--families", default="snap,medicaid,soi") parser.add_argument("--geographies", default="state,cd") - parser.add_argument("--soi-mode", choices=["full", "totals"], default="full") + parser.add_argument( + "--soi-mode", + choices=SOI_MODES, + default=DEFAULT_SOI_MODE, + help=( + "State SOI target surface for --stage materialize. 'state' " + "(default) is Build O's contract: every state-geography SOI spec " + "outside the congressional-district file. 'totals' drops every " + "soi_fiscal_distribution spec (no state AGI, income-tax or EITC " + "total); 'full' keeps every state-bearing spec, including the " + "district file, and needs a much larger dense admin matrix " + "(contents and sizes in docs/us-acs-local-soi-target-surface.md). " + "Later stages use the mode the checkpoint recorded." + ), + ) parser.add_argument("--epochs", type=int, default=800) parser.add_argument("--epoch-batch", type=int, default=400) parser.add_argument("--max-weight-ratio", type=float, default=5.0)